[llvm] [AMDGPU] PeepholeOptimizer - continue search next new source if a new source found is a subreg (PR #212563)
Yoonseo Choi via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 00:22:20 PDT 2026
https://github.com/yoonseoch updated https://github.com/llvm/llvm-project/pull/212563
>From 4ae30f8636dde32e28f182f4f7b5e798433ee18f Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 17 Jul 2026 18:50:44 +0000
Subject: [PATCH 1/9] Peephole-opt
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 34 +++++++++++++++++++++++++-
1 file changed, 33 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index d46ace0bdd748..77092e69c3abb 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1022,7 +1022,14 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
RegSubRegPair CurSrcPair = RegSubReg;
SmallVector<RegSubRegPair, 4> SrcToLook = {CurSrcPair};
+ LLVM_DEBUG(dbgs() << "EXPenter reg="
+ << printReg(RegSubReg.Reg, TRI, RegSubReg.SubReg) << "\n");
unsigned PHICount = 0;
+ // EXPERIMENT: remember the last suitable source so we can keep tracing past
+ // it (through REG_SEQUENCE, etc.) and still fall back if the deeper trace
+ // dead-ends.
+ bool FoundSuitable = false;
+ RegSubRegPair SuitablePair = RegSubReg;
do {
CurSrcPair = SrcToLook.pop_back_val();
// As explained above, do not handle physical registers
@@ -1036,8 +1043,18 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
while (true) {
ValueTrackerResult Res = ValTracker.getNextSource();
// Abort at the end of a chain (without finding a suitable source).
- if (!Res.isValid())
+ if (!Res.isValid()) {
+ // EXPERIMENT: if we already passed a suitable source while trying to
+ // reach a deeper one, fall back to it instead of aborting.
+ if (FoundSuitable) {
+ LLVM_DEBUG(dbgs() << "EXP fallback " << printReg(SuitablePair.Reg,
+ TRI, SuitablePair.SubReg) << " (orig "
+ << printReg(Reg, TRI) << ")\n");
+ CurSrcPair = SuitablePair;
+ break;
+ }
return false;
+ }
// Insert the Def -> Use entry for the recently found source.
auto [InsertPt, WasInserted] = RewriteMap.try_emplace(CurSrcPair, Res);
@@ -1090,7 +1107,22 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
if (PHICount > 0 && CurSrcPair.SubReg != 0)
continue;
+ // EXPERIMENT: don't stop at the first suitable source if it is still a
+ // subregister; keep tracing to try to reach a deeper source. Remember it
+ // as a fallback.
+ if (CurSrcPair.SubReg != 0) {
+ LLVM_DEBUG(dbgs() << "EXP continue " << printReg(CurSrcPair.Reg, TRI,
+ CurSrcPair.SubReg) << " (orig "
+ << printReg(Reg, TRI) << ")\n");
+ SuitablePair = CurSrcPair;
+ FoundSuitable = true;
+ continue;
+ }
+
// We found a suitable source, and are done with this chain.
+ LLVM_DEBUG(dbgs() << "EXP final " << printReg(CurSrcPair.Reg, TRI,
+ CurSrcPair.SubReg) << " (orig " << printReg(Reg, TRI)
+ << ")\n");
break;
}
} while (!SrcToLook.empty());
>From a43ac0701758aaf264f4bf52d2dd13dab7b6d5a4 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 17 Jul 2026 18:51:20 +0000
Subject: [PATCH 2/9] peephole-opt
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 77092e69c3abb..4ba76ca942a6c 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1047,9 +1047,10 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// EXPERIMENT: if we already passed a suitable source while trying to
// reach a deeper one, fall back to it instead of aborting.
if (FoundSuitable) {
- LLVM_DEBUG(dbgs() << "EXP fallback " << printReg(SuitablePair.Reg,
- TRI, SuitablePair.SubReg) << " (orig "
- << printReg(Reg, TRI) << ")\n");
+ LLVM_DEBUG(dbgs()
+ << "EXP fallback "
+ << printReg(SuitablePair.Reg, TRI, SuitablePair.SubReg)
+ << " (orig " << printReg(Reg, TRI) << ")\n");
CurSrcPair = SuitablePair;
break;
}
@@ -1111,18 +1112,18 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// subregister; keep tracing to try to reach a deeper source. Remember it
// as a fallback.
if (CurSrcPair.SubReg != 0) {
- LLVM_DEBUG(dbgs() << "EXP continue " << printReg(CurSrcPair.Reg, TRI,
- CurSrcPair.SubReg) << " (orig "
- << printReg(Reg, TRI) << ")\n");
+ LLVM_DEBUG(dbgs() << "EXP continue "
+ << printReg(CurSrcPair.Reg, TRI, CurSrcPair.SubReg)
+ << " (orig " << printReg(Reg, TRI) << ")\n");
SuitablePair = CurSrcPair;
FoundSuitable = true;
continue;
}
// We found a suitable source, and are done with this chain.
- LLVM_DEBUG(dbgs() << "EXP final " << printReg(CurSrcPair.Reg, TRI,
- CurSrcPair.SubReg) << " (orig " << printReg(Reg, TRI)
- << ")\n");
+ LLVM_DEBUG(dbgs() << "EXP final "
+ << printReg(CurSrcPair.Reg, TRI, CurSrcPair.SubReg)
+ << " (orig " << printReg(Reg, TRI) << ")\n");
break;
}
} while (!SrcToLook.empty());
>From 077f75360b92f92af80eb46c8a97ec28db5a02d5 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Sat, 25 Jul 2026 06:04:57 +0000
Subject: [PATCH 3/9] Add fallback logic
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 33 +++++++++++++++++++++++---
1 file changed, 30 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 4ba76ca942a6c..fca2a670ea974 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1030,11 +1030,25 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// dead-ends.
bool FoundSuitable = false;
RegSubRegPair SuitablePair = RegSubReg;
+ // Fall back to the last suitable (sub)register source we remembered while
+ // trying to reach a deeper one. Drop the speculative map edge leaving it so
+ // getNewSource() terminates exactly at the fallback, and stop exploring any
+ // remaining (PHI) work items.
+ auto FallBackToSuitable = [&]() {
+ CurSrcPair = SuitablePair;
+ RewriteMap.erase(SuitablePair);
+ SrcToLook.clear();
+ };
do {
CurSrcPair = SrcToLook.pop_back_val();
// As explained above, do not handle physical registers
- if (CurSrcPair.Reg.isPhysical())
+ if (CurSrcPair.Reg.isPhysical()) {
+ if (FoundSuitable) {
+ FallBackToSuitable();
+ break;
+ }
return false;
+ }
ValueTracker ValTracker(CurSrcPair.Reg, CurSrcPair.SubReg, *MRI, TII);
@@ -1051,7 +1065,7 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
<< "EXP fallback "
<< printReg(SuitablePair.Reg, TRI, SuitablePair.SubReg)
<< " (orig " << printReg(Reg, TRI) << ")\n");
- CurSrcPair = SuitablePair;
+ FallBackToSuitable();
break;
}
return false;
@@ -1069,6 +1083,10 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
if (CurSrcRes.getNumSources() > 1) {
LLVM_DEBUG(dbgs()
<< "findNextSource: found PHI cycle, aborting...\n");
+ if (FoundSuitable) {
+ FallBackToSuitable();
+ break;
+ }
return false;
}
break;
@@ -1081,6 +1099,10 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
PHICount++;
if (PHICount >= RewritePHILimit) {
LLVM_DEBUG(dbgs() << "findNextSource: PHI limit reached\n");
+ if (FoundSuitable) {
+ FallBackToSuitable();
+ break;
+ }
return false;
}
@@ -1094,8 +1116,13 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// constraints to the register allocator. Moreover, if we want to extend
// the live-range of a physical register, unlike SSA virtual register,
// we will have to check that they aren't redefine before the related use.
- if (CurSrcPair.Reg.isPhysical())
+ if (CurSrcPair.Reg.isPhysical()) {
+ if (FoundSuitable) {
+ FallBackToSuitable();
+ break;
+ }
return false;
+ }
// Keep following the chain if the value isn't any better yet.
const TargetRegisterClass *SrcRC = MRI->getRegClass(CurSrcPair.Reg);
>From 9abff682b139ef47595aa90dc436941c4ed47703 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 7 Aug 2026 18:54:39 +0000
Subject: [PATCH 4/9] Update lit tests for peephole-opt changes
Regenerated with utils/update_any_test_checks.py after rebasing onto main.
spill-vgpr.ll is intentionally left untouched: it is hand-written, and the
GFX900 ScratchSize check in max_256_vgprs_spill_9x32_2bb still records main's
132 so the increase to 148 caused by the findNextSource change stays visible.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll | 190 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 14 +-
.../atomic_optimizations_global_pointer.ll | 2280 +++++++++--------
.../atomic_optimizations_local_pointer.ll | 24 +-
.../branch-folding-implicit-def-subreg.ll | 416 ++-
.../AMDGPU/buffer-fat-pointers-memcpy.ll | 248 +-
llvm/test/CodeGen/AMDGPU/bypass-div.ll | 1593 ++++++------
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 68 +-
llvm/test/CodeGen/AMDGPU/idiv-licm.ll | 50 +-
...issue139317-bad-opsel-reg-sequence-fold.ll | 4 +-
llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll | 4 +-
llvm/test/CodeGen/AMDGPU/itofp.i128.ll | 66 +-
.../loop-live-out-copy-undef-subrange.ll | 6 +-
.../CodeGen/AMDGPU/memintrinsic-unroll.ll | 657 ++---
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 1510 ++++++-----
.../AMDGPU/memset-param-combinations.ll | 821 ++----
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 96 +-
.../peephole-opt-fold-reg-sequence-subreg.mir | 2 +-
.../AMDGPU/peephole-opt-regseq-removal.mir | 10 +-
.../AMDGPU/promote-constOffset-to-imm.ll | 100 +-
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 71 +-
...rename-independent-subregs-unused-lanes.ll | 58 +-
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 105 +-
.../AMDGPU/sgpr-scavenge-fi-stack-id.ll | 10 +-
.../splitkit-getsubrangeformask-phi-extend.ll | 2228 ++++++----------
llvm/test/CodeGen/AMDGPU/srem64.ll | 91 +-
llvm/test/CodeGen/AMDGPU/swdev373493.ll | 31 +-
.../test/CodeGen/AMDGPU/vni8-across-blocks.ll | 323 ++-
.../X86/vector-shift-by-select-loop.ll | 1 -
30 files changed, 5167 insertions(+), 5922 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index f77a4940360e5..1fa07b4b81702 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -952,12 +952,12 @@ define void @flat_atomic_xchg_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_xchg_i64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -966,23 +966,23 @@ define void @flat_atomic_xchg_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB14_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB14_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB14_4: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(2)
@@ -1044,12 +1044,12 @@ define void @flat_atomic_xchg_i64_ret_av_v(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_xchg_i64_ret_av_v:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -1058,23 +1058,23 @@ define void @flat_atomic_xchg_i64_ret_av_v(ptr %ptr) #0 {
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB15_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB15_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB15_4: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(2)
@@ -1328,12 +1328,12 @@ define void @flat_atomic_xchg_i64_ret_v_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_xchg_i64_ret_v_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -1342,23 +1342,23 @@ define void @flat_atomic_xchg_i64_ret_v_av(ptr %ptr) #0 {
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB18_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB18_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB18_4: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(2)
@@ -6512,35 +6512,35 @@ define void @flat_atomic_add_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_add_i64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB90_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB90_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB90_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
-; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
-; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, v0, v4
+; GFX90A-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc
+; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB90_4: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -6697,35 +6697,35 @@ define void @flat_atomic_sub_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_sub_i64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB92_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB92_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB92_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
-; GFX90A-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
-; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v4
+; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v5, vcc
+; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB92_4: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8986,28 +8986,28 @@ define void @flat_atomic_usub_sat_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_usub_sat_i64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[4:5]
+; GFX90A-NEXT: ; def v[6:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB114_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB114_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
-; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
+; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v6
+; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v7, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -9016,20 +9016,20 @@ define void @flat_atomic_usub_sat_i64_ret_av_av(ptr %ptr) #0 {
; GFX90A-NEXT: s_cbranch_execnz .LBB114_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX90A-NEXT: .LBB114_4: ; %Flow3
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB114_6
; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v4, vcc
; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v4
+; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v6
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v5, vcc
+; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v7, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
@@ -10520,31 +10520,31 @@ define void @flat_atomic_fmax_f64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_fmax_f64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB132_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB132_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB132_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
@@ -10708,31 +10708,31 @@ define void @flat_atomic_fmin_f64_ret_av_av(ptr %ptr) #0 {
; GFX90A-LABEL: flat_atomic_fmin_f64_ret_av_av:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:3]
+; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB134_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[4:5], v[2:3] glc
+; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3], v[4:5] glc
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: .LBB134_2: ; %Flow
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB134_4
; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private
-; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
+; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX90A-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 275118e33eeab..26821b059cb6c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -8654,12 +8654,12 @@ define inreg <4 x bfloat> @bitcast_v2i32_to_v4bf16_scalar(<2 x i32> inreg %a, i3
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB65_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_add_i32 s4, s16, 3
-; SI-NEXT: s_add_i32 s5, s17, 3
-; SI-NEXT: s_and_b32 s7, s5, 0xffff0000
-; SI-NEXT: s_lshl_b32 s6, s5, 16
-; SI-NEXT: s_and_b32 s9, s4, 0xffff0000
-; SI-NEXT: s_lshl_b32 s8, s4, 16
+; SI-NEXT: s_add_i32 s16, s16, 3
+; SI-NEXT: s_add_i32 s17, s17, 3
+; SI-NEXT: s_and_b32 s7, s17, 0xffff0000
+; SI-NEXT: s_lshl_b32 s6, s17, 16
+; SI-NEXT: s_and_b32 s9, s16, 0xffff0000
+; SI-NEXT: s_lshl_b32 s8, s16, 16
; SI-NEXT: .LBB65_5: ; %end
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
@@ -14482,7 +14482,7 @@ define <8 x i8> @bitcast_v4i16_to_v8i8(<4 x i16> %a, i32 %b) #0 {
; VI-NEXT: v_lshrrev_b32_e32 v4, 8, v0
; VI-NEXT: v_mov_b32_e32 v9, v0
; VI-NEXT: v_mov_b32_e32 v8, v1
-; VI-NEXT: ; implicit-def: $vgpr0_vgpr1
+; VI-NEXT: ; implicit-def: $vgpr1
; VI-NEXT: ; %bb.2: ; %Flow
; VI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; VI-NEXT: s_cbranch_execz .LBB96_4
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 0fa5e932e7cea..0b6e00ef0ac9c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -4671,7 +4671,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-LABEL: sub_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
@@ -4681,20 +4681,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
-; GFX7LESS-NEXT: s_mul_i32 s11, s10, s6
+; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s13, s12, s5
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
-; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s11, v4
+; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s13, v4
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v0, v3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v4
@@ -4702,15 +4702,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: .LBB7_4: ; %Flow3
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v1, s10, v2
+; GFX7LESS-NEXT: v_mul_lo_u32 v1, s12, v2
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -6444,10 +6444,10 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
+; GFX7LESS-NEXT: s_mul_i32 s12, s12, 5
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
@@ -6503,10 +6503,10 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s12, s6, 5
+; GFX8-NEXT: s_mul_i32 s12, s12, 5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
@@ -6560,10 +6560,10 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s12, s6, 5
+; GFX9-NEXT: s_mul_i32 s12, s12, 5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
@@ -6617,9 +6617,9 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s12, s6, 5
+; GFX1064-NEXT: s_mul_i32 s12, s12, 5
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -6674,9 +6674,9 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s6
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s6, 5
+; GFX1032-NEXT: s_mul_i32 s10, s10, 5
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v0, s4
@@ -6731,9 +6731,9 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s12, s6, 5
+; GFX1164-NEXT: s_mul_i32 s12, s12, 5
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -6796,9 +6796,9 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s6
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s6, 5
+; GFX1132-NEXT: s_mul_i32 s10, s10, 5
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
@@ -7018,23 +7018,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX7LESS-NEXT: s_mul_i32 s3, s9, s2
-; GFX7LESS-NEXT: s_mul_i32 s2, s8, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, s2
-; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s3, v0
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_mul_i32 s7, s9, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
+; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v9, v0
@@ -7050,11 +7050,11 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
@@ -7150,8 +7150,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_mul_i32 s7, s9, s6
-; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
-; GFX9-NEXT: s_add_i32 s14, s12, s7
+; GFX9-NEXT: s_mul_hi_u32 s14, s8, s6
+; GFX9-NEXT: s_add_i32 s14, s14, s7
; GFX9-NEXT: s_mul_i32 s15, s8, s6
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -9348,7 +9348,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-LABEL: uniform_or_i8:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s14, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -9361,35 +9361,35 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
-; GFX7LESS-NEXT: s_and_b32 s0, s12, 0xff
-; GFX7LESS-NEXT: s_lshl_b32 s14, s0, s13
-; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_and_b32 s0, s14, 0xff
+; GFX7LESS-NEXT: s_lshl_b32 s15, s0, s10
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7LESS-NEXT: .LBB12_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
@@ -9400,7 +9400,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-LABEL: uniform_or_i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -9413,34 +9413,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
-; GFX8-NEXT: s_and_b32 s0, s12, 0xff
-; GFX8-NEXT: s_lshl_b32 s14, s0, s13
-; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_lshl_b32 s10, s0, 3
+; GFX8-NEXT: s_and_b32 s0, s14, 0xff
+; GFX8-NEXT: s_lshl_b32 s15, s0, s10
+; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB12_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: .LBB12_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
@@ -9451,7 +9451,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-LABEL: uniform_or_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -9464,34 +9464,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
-; GFX9-NEXT: s_and_b32 s0, s12, 0xff
-; GFX9-NEXT: s_lshl_b32 s14, s0, s13
-; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_lshl_b32 s10, s0, 3
+; GFX9-NEXT: s_and_b32 s0, s14, 0xff
+; GFX9-NEXT: s_lshl_b32 s15, s0, s10
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: .LBB12_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v0, s14
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
@@ -9503,7 +9503,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -9516,34 +9516,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1064-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1064-NEXT: .LBB12_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
@@ -9565,18 +9565,19 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_and_b32 s6, s10, 3
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, s1, 0xff
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s5, s10, 3
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1032-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -9604,7 +9605,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -9618,10 +9619,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -9629,26 +9630,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-TRUE16-NEXT: .LBB12_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
@@ -9659,7 +9660,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -9673,10 +9674,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -9684,26 +9685,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-FAKE16-NEXT: .LBB12_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
@@ -9726,19 +9727,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -9779,19 +9782,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -9820,7 +9825,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -9834,10 +9839,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -9845,26 +9850,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-TRUE16-NEXT: .LBB12_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -9877,7 +9882,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -9891,10 +9896,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -9902,26 +9907,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-FAKE16-NEXT: .LBB12_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -9946,19 +9951,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -10001,19 +10008,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -10260,7 +10269,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-LABEL: uniform_add_i8:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
@@ -10271,45 +10280,46 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s4
+; GFX7LESS-NEXT: s_mul_i32 s6, s12, s4
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s11, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX7LESS-NEXT: s_and_b32 s2, s6, 0xff
-; GFX7LESS-NEXT: s_not_b32 s13, s12
-; GFX7LESS-NEXT: s_lshl_b32 s14, s2, s11
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff
+; GFX7LESS-NEXT: s_not_b32 s14, s13
+; GFX7LESS-NEXT: s_lshl_b32 s15, s6, s2
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s13, v0
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX7LESS-NEXT: .LBB13_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xff
+; GFX7LESS-NEXT: s_and_b32 s5, s12, 0xff
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -10320,7 +10330,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-LABEL: uniform_add_i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10331,39 +10341,40 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_mul_i32 s6, s12, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s11, s2, 3
-; GFX8-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX8-NEXT: s_and_b32 s2, s6, 0xff
-; GFX8-NEXT: s_not_b32 s13, s12
-; GFX8-NEXT: s_lshl_b32 s14, s2, s11
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_not_b32 s14, s13
+; GFX8-NEXT: s_lshl_b32 s15, s6, s2
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
-; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s15, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s13, v0
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB13_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX8-NEXT: .LBB13_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_readfirstlane_b32 s4, v0
@@ -10371,14 +10382,14 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: v_mad_u16 v0, s12, v4, v0
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_add_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10389,38 +10400,39 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_mul_i32 s6, s12, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s11, s2, 3
-; GFX9-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX9-NEXT: s_and_b32 s2, s6, 0xff
-; GFX9-NEXT: s_not_b32 s13, s12
-; GFX9-NEXT: s_lshl_b32 s14, s2, s11
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_not_b32 s14, s13
+; GFX9-NEXT: s_lshl_b32 s15, s6, s2
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
-; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_add_u32_e32 v0, s15, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX9-NEXT: .LBB13_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -10428,7 +10440,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, s12, v4, v0
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -10436,7 +10448,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10449,42 +10461,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
-; GFX1064-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1064-NEXT: s_not_b32 s13, s12
-; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1064-NEXT: s_not_b32 s14, s13
+; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: .LBB13_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX1064-NEXT: s_endpgm
@@ -10506,23 +10519,24 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_mul_i32 s6, s8, s6
-; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1032-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1032-NEXT: s_not_b32 s11, s3
-; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_not_b32 s12, s11
+; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s11, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -10549,7 +10563,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -10563,26 +10577,27 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -10590,20 +10605,20 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: .LBB13_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
; GFX1164-TRUE16-NEXT: s_endpgm
@@ -10612,7 +10627,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -10626,26 +10641,27 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
@@ -10653,20 +10669,20 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: .LBB13_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
; GFX1164-FAKE16-NEXT: s_endpgm
@@ -10689,25 +10705,26 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -10749,25 +10766,26 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -10795,7 +10813,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -10809,40 +10827,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: .LBB13_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -10850,7 +10871,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1264-TRUE16-NEXT: s_endpgm
@@ -10859,7 +10880,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -10873,40 +10894,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: .LBB13_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -10914,7 +10938,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1264-FAKE16-NEXT: s_endpgm
@@ -10937,28 +10961,29 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
+; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -11001,28 +11026,29 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
+; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -11296,40 +11322,41 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s8, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX7LESS-NEXT: s_not_b32 s9, s2
-; GFX7LESS-NEXT: s_and_b32 s2, s6, 0xff
-; GFX7LESS-NEXT: s_lshl_b32 s10, s2, s8
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s2, 0xff, s10
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff
+; GFX7LESS-NEXT: s_not_b32 s2, s2
+; GFX7LESS-NEXT: s_lshl_b32 s11, s6, s10
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
-; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s11, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1
; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
;
@@ -11337,39 +11364,40 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s8, s2, 3
-; GFX8-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX8-NEXT: s_not_b32 s9, s2
-; GFX8-NEXT: s_and_b32 s2, s6, 0xff
-; GFX8-NEXT: s_lshl_b32 s10, s2, s8
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s10, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xff, s10
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: s_lshl_b32 s11, s6, s10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -11377,39 +11405,40 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s8, s2, 3
-; GFX9-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX9-NEXT: s_not_b32 s9, s2
-; GFX9-NEXT: s_and_b32 s2, s6, 0xff
-; GFX9-NEXT: s_lshl_b32 s10, s2, s8
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s10, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xff, s10
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_lshl_b32 s11, s6, s10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB14_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -11418,36 +11447,37 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1064-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1064-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1064-NEXT: s_not_b32 s10, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -11463,19 +11493,20 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1032-NEXT: s_not_b32 s8, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -11498,38 +11529,39 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_and_b32 s4, s2, -4
; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1164-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1164-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1164-NEXT: s_not_b32 s10, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_1
; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 0
@@ -11545,20 +11577,21 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1132-NEXT: s_and_b32 s4, s2, -4
; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1132-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1132-NEXT: s_not_b32 s8, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -11581,38 +11614,39 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_and_b32 s4, s2, -4
; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1264-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1264-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1264-NEXT: s_not_b32 s10, s7
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-NEXT: s_mov_b32 s6, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1264-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_cbranch_execnz .LBB14_1
; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null
@@ -11628,20 +11662,21 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1232-NEXT: s_and_b32 s4, s2, -4
; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1232-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1232-NEXT: s_not_b32 s8, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-NEXT: s_mov_b32 s6, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -11748,7 +11783,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-LABEL: uniform_or_i16:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s14, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -11761,35 +11796,35 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
-; GFX7LESS-NEXT: s_and_b32 s0, s12, 0xffff
-; GFX7LESS-NEXT: s_lshl_b32 s14, s0, s13
-; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_and_b32 s0, s14, 0xffff
+; GFX7LESS-NEXT: s_lshl_b32 s15, s0, s10
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s13, 16
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s10, 16
; GFX7LESS-NEXT: .LBB15_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
@@ -11800,7 +11835,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-LABEL: uniform_or_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -11813,34 +11848,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s12
-; GFX8-NEXT: s_lshl_b32 s14, s0, s13
-; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_lshl_b32 s10, s0, 3
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX8-NEXT: s_lshl_b32 s15, s0, s10
+; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB15_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: .LBB15_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
@@ -11851,7 +11886,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-LABEL: uniform_or_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -11864,34 +11899,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
-; GFX9-NEXT: s_and_b32 s0, 0xffff, s12
-; GFX9-NEXT: s_lshl_b32 s14, s0, s13
-; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_lshl_b32 s10, s0, 3
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX9-NEXT: s_lshl_b32 s15, s0, s10
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: .LBB15_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v0, s14
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
@@ -11903,7 +11938,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -11916,34 +11951,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1064-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1064-NEXT: .LBB15_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
@@ -11965,18 +12000,19 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_and_b32 s6, s10, 3
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s5, s10, 3
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1032-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -12004,7 +12040,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12018,10 +12054,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -12029,26 +12065,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-TRUE16-NEXT: .LBB15_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
@@ -12059,7 +12095,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12073,10 +12109,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -12084,26 +12120,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-FAKE16-NEXT: .LBB15_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
@@ -12126,19 +12162,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12179,19 +12217,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -12220,7 +12260,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12234,10 +12274,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -12245,26 +12285,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-TRUE16-NEXT: .LBB15_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -12277,7 +12317,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12291,10 +12331,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -12302,26 +12342,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-FAKE16-NEXT: .LBB15_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -12346,19 +12386,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -12401,19 +12443,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -12660,7 +12704,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS-LABEL: uniform_add_i16:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
@@ -12671,45 +12715,46 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s4
+; GFX7LESS-NEXT: s_mul_i32 s6, s12, s4
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s11, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX7LESS-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX7LESS-NEXT: s_not_b32 s13, s12
-; GFX7LESS-NEXT: s_lshl_b32 s14, s2, s11
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX7LESS-NEXT: s_not_b32 s14, s13
+; GFX7LESS-NEXT: s_lshl_b32 s15, s6, s2
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s13, v0
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s11, 16
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s2, 16
; GFX7LESS-NEXT: .LBB16_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xffff
+; GFX7LESS-NEXT: s_and_b32 s5, s12, 0xffff
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -12720,7 +12765,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-LABEL: uniform_add_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -12731,39 +12776,40 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_mul_i32 s6, s12, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s11, s2, 3
-; GFX8-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX8-NEXT: s_not_b32 s13, s12
-; GFX8-NEXT: s_lshl_b32 s14, s2, s11
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX8-NEXT: s_not_b32 s14, s13
+; GFX8-NEXT: s_lshl_b32 s15, s6, s2
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
-; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s15, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s13, v0
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB16_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX8-NEXT: .LBB16_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_readfirstlane_b32 s4, v0
@@ -12771,14 +12817,14 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: v_mad_u16 v0, s12, v4, v0
; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_add_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -12789,38 +12835,39 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_mul_i32 s6, s12, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s11, s2, 3
-; GFX9-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX9-NEXT: s_not_b32 s13, s12
-; GFX9-NEXT: s_lshl_b32 s14, s2, s11
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX9-NEXT: s_not_b32 s14, s13
+; GFX9-NEXT: s_lshl_b32 s15, s6, s2
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
-; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_add_u32_e32 v0, s15, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX9-NEXT: .LBB16_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -12828,7 +12875,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, s12, v4, v0
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -12836,7 +12883,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -12849,42 +12896,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
-; GFX1064-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1064-NEXT: s_not_b32 s13, s12
-; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1064-NEXT: s_not_b32 s14, s13
+; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: .LBB16_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX1064-NEXT: s_endpgm
@@ -12906,23 +12954,24 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_mul_i32 s6, s8, s6
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1032-NEXT: s_not_b32 s11, s3
-; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_not_b32 s12, s11
+; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s11, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -12949,7 +12998,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -12963,26 +13012,27 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -12990,20 +13040,20 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: .LBB16_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX1164-TRUE16-NEXT: s_endpgm
@@ -13012,7 +13062,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -13026,26 +13076,27 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
@@ -13053,20 +13104,20 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: .LBB16_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX1164-FAKE16-NEXT: s_endpgm
@@ -13089,25 +13140,26 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -13149,25 +13201,26 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -13195,7 +13248,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -13209,40 +13262,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: .LBB16_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -13250,7 +13306,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1264-TRUE16-NEXT: s_endpgm
@@ -13259,7 +13315,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -13273,40 +13329,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: .LBB16_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -13314,7 +13373,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1264-FAKE16-NEXT: s_endpgm
@@ -13337,28 +13396,29 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
+; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -13401,28 +13461,29 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
+; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -13696,40 +13757,41 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s8, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX7LESS-NEXT: s_not_b32 s9, s2
-; GFX7LESS-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX7LESS-NEXT: s_lshl_b32 s10, s2, s8
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
+; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX7LESS-NEXT: s_not_b32 s2, s2
+; GFX7LESS-NEXT: s_lshl_b32 s11, s6, s10
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
-; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s11, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1
; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
;
@@ -13737,39 +13799,40 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s8, s2, 3
-; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX8-NEXT: s_not_b32 s9, s2
-; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s10, s2, s8
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s10, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: s_lshl_b32 s11, s6, s10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -13777,39 +13840,40 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s8, s2, 3
-; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX9-NEXT: s_not_b32 s9, s2
-; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX9-NEXT: s_lshl_b32 s10, s2, s8
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s10, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_lshl_b32 s11, s6, s10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB17_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -13818,36 +13882,37 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1064-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1064-NEXT: s_not_b32 s10, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -13863,19 +13928,20 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1032-NEXT: s_not_b32 s8, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -13898,38 +13964,39 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_and_b32 s4, s2, -4
; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1164-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1164-NEXT: s_not_b32 s10, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_1
; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -13945,20 +14012,21 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-NEXT: s_and_b32 s4, s2, -4
; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1132-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1132-NEXT: s_not_b32 s8, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -13981,38 +14049,39 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_and_b32 s4, s2, -4
; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1264-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1264-NEXT: s_not_b32 s10, s7
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-NEXT: s_mov_b32 s6, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1264-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_cbranch_execnz .LBB17_1
; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -14028,20 +14097,21 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-NEXT: s_and_b32 s4, s2, -4
; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1232-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1232-NEXT: s_not_b32 s8, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-NEXT: s_mov_b32 s6, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -14155,13 +14225,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -14200,13 +14271,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
; GFX8-NEXT: s_lshl_b32 s10, s2, 3
; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX8-NEXT: s_not_b32 s2, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -14241,13 +14313,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
; GFX9-NEXT: s_lshl_b32 s10, s2, 3
; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX9-NEXT: s_not_b32 s2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -14275,38 +14348,39 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1064-NEXT: s_not_b32 s10, s2
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1064-NEXT: s_not_b32 s11, s6
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1064-NEXT: v_add_f16_e32 v0, s8, v0
-; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1064-NEXT: v_add_f16_e32 v0, s10, v0
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB18_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -14323,19 +14397,20 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s6
+; GFX1032-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1032-NEXT: s_not_b32 s10, s6
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1032-NEXT: v_add_f16_e32 v0, s8, v0
; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -14357,44 +14432,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1164-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1164-TRUE16-NEXT: s_not_b32 s11, s6
+; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s10, v0.l
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -14404,44 +14480,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1164-FAKE16-NEXT: s_not_b32 s10, s2
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1164-FAKE16-NEXT: s_not_b32 s11, s6
+; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1164-FAKE16-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1164-FAKE16-NEXT: v_add_f16_e32 v0, s10, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -14458,14 +14535,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s10, s6
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14475,7 +14553,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -14504,14 +14582,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s10, s6
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14521,7 +14600,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -14543,44 +14622,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1264-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1264-TRUE16-NEXT: s_not_b32 s11, s6
+; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s10, v0.l
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
-; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -14590,44 +14670,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1264-FAKE16-NEXT: s_not_b32 s10, s2
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1264-FAKE16-NEXT: s_not_b32 s11, s6
+; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1264-FAKE16-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1264-FAKE16-NEXT: v_add_f16_e32 v0, s10, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -14644,14 +14725,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1232-TRUE16-NEXT: s_not_b32 s10, s6
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14661,7 +14743,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14690,14 +14772,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1232-FAKE16-NEXT: s_not_b32 s10, s6
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14707,7 +14790,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -14922,14 +15005,15 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX7LESS-NEXT: s_lshl_b32 s11, s6, 16
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -14967,16 +15051,17 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s3, s2, 3
-; GFX8-NEXT: s_lshl_b32 s10, s3, 3
-; GFX8-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s7, s2, 3
+; GFX8-NEXT: s_lshl_b32 s10, s7, 3
; GFX8-NEXT: s_lshl_b32 s2, s6, 16
-; GFX8-NEXT: s_not_b32 s3, s3
+; GFX8-NEXT: s_lshl_b32 s6, 0xffff, s10
+; GFX8-NEXT: s_not_b32 s11, s6
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
@@ -14988,7 +15073,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, s3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s11, v1
; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
@@ -15013,31 +15098,32 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[8:9], 0
-; GFX9-NEXT: s_movk_i32 s11, 0x7fff
+; GFX9-NEXT: s_movk_i32 s12, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s3, s2, 3
-; GFX9-NEXT: s_lshl_b32 s10, s3, 3
-; GFX9-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s7, s2, 3
+; GFX9-NEXT: s_lshl_b32 s10, s7, 3
; GFX9-NEXT: s_lshl_b32 s2, s6, 16
-; GFX9-NEXT: s_not_b32 s3, s3
+; GFX9-NEXT: s_lshl_b32 s6, 0xffff, s10
+; GFX9-NEXT: s_not_b32 s11, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_f32_e32 v0, s2, v0
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_add3_u32 v2, v2, v0, s11
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s12
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -15060,43 +15146,44 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1064-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1064-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_not_b32 s9, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_not_b32 s10, s7
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX1064-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX1064-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1064-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1064-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX1064-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1064-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB19_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -15107,42 +15194,43 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s8, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1032-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_not_b32 s8, s3
-; GFX1032-NEXT: s_mov_b32 s3, 0
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_not_b32 s9, s7
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX1032-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1032-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1032-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1032-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1032-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1032-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v2
-; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_cbranch_execnz .LBB19_1
; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: s_mov_b32 s2, -1
@@ -15154,28 +15242,29 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_and_b32 s4, s2, -4
; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1164-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1164-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1164-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_not_b32 s9, s2
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_not_b32 s10, s7
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .p2align 6
; GFX1164-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1164-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1164-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1164-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX1164-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1164-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -15184,9 +15273,9 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1164-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1164-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
@@ -15194,13 +15283,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v2
; GFX1164-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_cbranch_execnz .LBB19_1
; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -15211,20 +15300,21 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-NEXT: s_mov_b32 s8, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_and_b32 s4, s2, -4
; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1132-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_not_b32 s8, s3
-; GFX1132-NEXT: s_mov_b32 s3, 0
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-NEXT: s_not_b32 s9, s7
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .p2align 6
; GFX1132-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -15232,7 +15322,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1132-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1132-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1132-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1132-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1132-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -15243,18 +15333,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1132-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v2
-; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1132-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1132-NEXT: s_cbranch_execnz .LBB19_1
; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132-NEXT: s_mov_b32 s2, -1
@@ -15266,27 +15356,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_and_b32 s4, s2, -4
; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1264-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1264-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1264-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1264-NEXT: s_mov_b32 s6, -1
-; GFX1264-NEXT: s_not_b32 s9, s2
-; GFX1264-NEXT: s_mov_b64 s[2:3], 0
-; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_not_b32 s10, s7
; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1264-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1264-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1264-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1264-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1264-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -15296,22 +15387,22 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1264-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1264-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1264-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_cbranch_execnz .LBB19_1
; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -15322,27 +15413,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s8, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_and_b32 s4, s2, -4
; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1232-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1232-NEXT: s_mov_b32 s6, -1
-; GFX1232-NEXT: s_not_b32 s8, s3
-; GFX1232-NEXT: s_mov_b32 s3, 0
-; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-NEXT: s_not_b32 s9, s7
; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1232-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1232-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1232-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1232-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1232-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -15354,18 +15446,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1232-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-NEXT: s_wait_loadcnt 0x0
; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1232-NEXT: v_mov_b32_e32 v1, v2
-; GFX1232-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1232-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1232-NEXT: s_cbranch_execnz .LBB19_1
; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
; GFX1232-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index ad7db0bf81dae..970f572a42b6d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -13390,9 +13390,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: max_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s0, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -13445,9 +13445,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: max_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX8_ITERATIVE-NEXT: s_mov_b32 s0, 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -13498,9 +13498,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: max_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX9_ITERATIVE-NEXT: s_mov_b32 s0, 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -15584,9 +15584,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: min_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s0, -1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -15639,9 +15639,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: min_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX8_ITERATIVE-NEXT: s_mov_b32 s0, -1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -15692,9 +15692,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: min_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX9_ITERATIVE-NEXT: s_mov_b32 s0, -1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17773,8 +17773,8 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: umax_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17827,8 +17827,8 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: umax_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17879,8 +17879,8 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: umax_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19950,8 +19950,8 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: umin_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20004,8 +20004,8 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: umin_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20056,8 +20056,8 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: umin_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index 11279b29b15b0..5752fde325d0c 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -43,31 +43,31 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: successors: %bb.3(0x80000000)
; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr40_sgpr41, $sgpr56, $sgpr57, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr4, $vgpr5
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
+ ; GFX90A-NEXT: renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
; GFX90A-NEXT: renamable $vgpr3 = IMPLICIT_DEF implicit-def $vgpr2
+ ; GFX90A-NEXT: renamable $vgpr23 = IMPLICIT_DEF implicit-def $vgpr22
+ ; GFX90A-NEXT: renamable $vgpr25 = IMPLICIT_DEF implicit-def $vgpr24
; GFX90A-NEXT: renamable $vgpr27 = IMPLICIT_DEF implicit-def $vgpr26
- ; GFX90A-NEXT: renamable $vgpr29 = IMPLICIT_DEF implicit-def $vgpr28
- ; GFX90A-NEXT: renamable $vgpr33 = IMPLICIT_DEF implicit-def $vgpr32
; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3.Flow17:
; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.57(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr6 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.57, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.4.bb15:
; GFX90A-NEXT: successors: %bb.35(0x40000000), %bb.5(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = nsw V_LSHLREV_B64_e64 2, $vgpr4_vgpr5, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr25, implicit $exec
; GFX90A-NEXT: renamable $vgpr46, renamable $vcc = V_ADD_CO_U32_e64 $sgpr24, $vgpr0, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr47, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr2, killed $vgpr1, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 2, $vgpr6, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 2, $vgpr30, implicit $exec
; GFX90A-NEXT: renamable $vgpr40, renamable $vcc = V_ADD_CO_U32_e64 $vgpr46, killed $vgpr0, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr41, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr47, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
@@ -75,7 +75,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.5:
; GFX90A-NEXT: successors: %bb.6(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
@@ -88,9 +88,9 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
@@ -99,31 +99,31 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr44_vgpr45 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr42_vgpr43 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.6.Flow20:
; GFX90A-NEXT: successors: %bb.7(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr26 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr28 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr33 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr17 = COPY killed renamable $sgpr18, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr29 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr13 = COPY killed renamable $sgpr18, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.7.Flow19:
; GFX90A-NEXT: successors: %bb.62(0x40000000), %bb.8(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
; GFX90A-NEXT: $sgpr18_sgpr19 = S_AND_SAVEEXEC_B64 $sgpr36_sgpr37, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -131,7 +131,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.8.Flow32:
; GFX90A-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr40_sgpr41, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -140,15 +140,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.9.bb89:
; GFX90A-NEXT: successors: %bb.10(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr13, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr12, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.10.Flow33:
; GFX90A-NEXT: successors: %bb.11(0x40000000), %bb.12(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr46_sgpr47, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -157,15 +157,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.11.bb84:
; GFX90A-NEXT: successors: %bb.12(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr55, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr54, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.12.Flow34:
; GFX90A-NEXT: successors: %bb.13(0x40000000), %bb.14(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr44_sgpr45, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -174,10 +174,10 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.13.bb79:
; GFX90A-NEXT: successors: %bb.14(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
- ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+ ; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.14.Flow35:
@@ -362,7 +362,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.35.bb20:
; GFX90A-NEXT: successors: %bb.37(0x40000000), %bb.36(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_SBYTE renamable $vgpr40_vgpr41, 1024, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i23)
; GFX90A-NEXT: renamable $vgpr42 = V_ADD_CO_U32_e32 1024, $vgpr40, implicit-def $vcc, implicit $exec
@@ -379,9 +379,9 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr43, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_LT_I16_e64 0, killed $vgpr0, implicit $exec
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
@@ -389,27 +389,27 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr56_vgpr57 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr44_vgpr45 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.37, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.36.Flow21:
; GFX90A-NEXT: successors: %bb.6(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
; GFX90A-NEXT: S_BRANCH %bb.6
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.37.bb27:
; GFX90A-NEXT: successors: %bb.39(0x40000000), %bb.38(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67, $sgpr48_sgpr49
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE renamable $vgpr40_vgpr41, 2048, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i30)
; GFX90A-NEXT: renamable $vgpr44 = V_ADD_CO_U32_e32 2048, $vgpr40, implicit-def $vcc, implicit $exec
@@ -419,29 +419,29 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr45, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr58_vgpr59 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr56_vgpr57 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr38_sgpr39 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.39, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.38.Flow22:
; GFX90A-NEXT: successors: %bb.36(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr38_sgpr39, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_XOR_B64 $exec, -1, implicit-def dead $scc
@@ -462,7 +462,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.39.bb34:
; GFX90A-NEXT: successors: %bb.41(0x40000000), %bb.40(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE renamable $vgpr40_vgpr41, 3072, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i37)
; GFX90A-NEXT: renamable $vgpr56 = V_ADD_CO_U32_e32 3072, $vgpr40, implicit-def $vcc, implicit $exec
@@ -472,28 +472,28 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr57, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr58_vgpr59 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.41, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.40.Flow23:
; GFX90A-NEXT: successors: %bb.38(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr68_sgpr69, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr68_sgpr69, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr40_sgpr41, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_XOR_B64 $exec, -1, implicit-def dead $scc
@@ -513,7 +513,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.41.bb41:
; GFX90A-NEXT: successors: %bb.46(0x40000000), %bb.42(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr66_sgpr67, $sgpr68_sgpr69
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr66_sgpr67, $sgpr68_sgpr69
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr58 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr1, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, $vgpr41, $vcc, 0, implicit $exec
@@ -523,27 +523,27 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
; GFX90A-NEXT: renamable $vgpr59, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr42_sgpr43 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.46, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.42.Flow24:
; GFX90A-NEXT: successors: %bb.40(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr42_sgpr43, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_XOR_B64 $exec, -1, implicit-def dead $scc
@@ -562,11 +562,11 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.43.bb55:
; GFX90A-NEXT: successors: %bb.48(0x40000000), %bb.44(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr20, 16, implicit-def $scc
- ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_CSELECT_B64 -1, 0, implicit killed $scc
- ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_XOR_B64 renamable $sgpr64_sgpr65, -1, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_CSELECT_B64 -1, 0, implicit killed $scc
+ ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_XOR_B64 renamable $sgpr66_sgpr67, -1, implicit-def dead $scc
; GFX90A-NEXT: renamable $vgpr62 = V_ADD_CO_U32_e32 6144, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr63, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr18_sgpr19, implicit-def dead $scc
@@ -574,27 +574,27 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.44:
; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $vgpr57, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $vgpr59, $vgpr40, $vgpr61, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22, $vgpr62, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr47, $vgpr56, $vgpr2, $vgpr4, $vgpr5, $vgpr6, $vgpr45, $vgpr46, $vgpr43, $vgpr44, $vgpr41, $vgpr42, $vgpr58, $vgpr60, $vgpr63
+ ; GFX90A-NEXT: liveins: $sgpr14, $vgpr57, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $vgpr41, $vgpr42, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22, $vgpr62, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr46, $vgpr56, $vgpr2, $vgpr4, $vgpr5, $vgpr45, $vgpr47, $vgpr43, $vgpr44, $vgpr58, $vgpr60, $vgpr63, $vgpr59, $vgpr61, $vgpr40
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.45.Flow26:
; GFX90A-NEXT: successors: %bb.47(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_XOR_B64 $exec, -1, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
@@ -610,7 +610,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.46.bb48:
; GFX90A-NEXT: successors: %bb.43(0x40000000), %bb.47(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr66_sgpr67, $sgpr58_sgpr59, $sgpr68_sgpr69, $sgpr64_sgpr65, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr66_sgpr67, $sgpr58_sgpr59, $sgpr68_sgpr69, $sgpr64_sgpr65, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr60 = V_ADD_CO_U32_e32 5120, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = COPY $vcc
@@ -622,26 +622,26 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr61, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $sgpr18_sgpr19, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr44_sgpr45 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.43, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.47.Flow25:
; GFX90A-NEXT: successors: %bb.42(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr58_sgpr59, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr58_sgpr59, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr44_sgpr45, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_XOR_B64 $exec, -1, implicit-def dead $scc
@@ -659,139 +659,140 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.48.bb63:
; GFX90A-NEXT: successors: %bb.50(0x40000000), %bb.49(0x40000000)
- ; GFX90A-NEXT: liveins: $vcc, $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $vcc, $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.50, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.49:
; GFX90A-NEXT: successors: %bb.44(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 -1
; GFX90A-NEXT: S_BRANCH %bb.44
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.50.bb68:
; GFX90A-NEXT: successors: %bb.54(0x40000000), %bb.51(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57:0x000000000000000F, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr6, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
; GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.54, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.51:
; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: S_BRANCH %bb.45
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.52.bb80:
; GFX90A-NEXT: successors: %bb.59(0x40000000), %bb.53(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: dead renamable $sgpr15 = S_BFE_U32 renamable $sgpr20, 65560, implicit-def $scc
- ; GFX90A-NEXT: renamable $vgpr10 = V_ADD_CO_U32_e32 4096, $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr11, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr8 = V_ADD_CO_U32_e32 4096, $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr9, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = COPY renamable $vgpr8_vgpr9, implicit $exec
; GFX90A-NEXT: S_CBRANCH_SCC0 %bb.59, implicit killed $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.53:
; GFX90A-NEXT: successors: %bb.61(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr36_sgpr37
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: S_BRANCH %bb.61
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.54.bb73:
; GFX90A-NEXT: successors: %bb.52(0x40000000), %bb.55(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr3 = FLAT_LOAD_UBYTE renamable $vgpr0_vgpr1, 2048, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i76)
- ; GFX90A-NEXT: renamable $vgpr8 = V_ADD_CO_U32_e32 2048, $vgpr0, implicit-def $vcc, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr6 = V_ADD_CO_U32_e32 2048, $vgpr0, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
- ; GFX90A-NEXT: renamable $vgpr9, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr7, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr3, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr62_sgpr63 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.52, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.55.Flow29:
; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr62_sgpr63, implicit-def $scc
; GFX90A-NEXT: S_BRANCH %bb.45
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.56.bb90:
; GFX90A-NEXT: successors: %bb.60(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: dead renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def $scc
+ ; GFX90A-NEXT: dead renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr23 = S_CSELECT_B32 1, 0, implicit killed $scc
; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr7 = COPY renamable $sgpr21, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr7, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr18_vgpr19 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $vgpr3 = COPY renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_LSHR_B64 killed renamable $sgpr56_sgpr57, 1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $vgpr18_vgpr19 = V_LSHRREV_B64_e64 1, $vgpr24_vgpr25, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr14_vgpr15 = V_LSHRREV_B64_e64 1, $vgpr20_vgpr21, implicit $exec
; GFX90A-NEXT: dead renamable $sgpr52_sgpr53 = S_AND_B64 renamable $sgpr12_sgpr13, $exec, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr15 = S_CSELECT_B32 1, 0, implicit killed $scc
- ; GFX90A-NEXT: renamable $vgpr14_vgpr15 = V_LSHRREV_B64_e64 1, $vgpr22_vgpr23, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = V_LSHRREV_B64_e64 1, $vgpr18_vgpr19, implicit $exec
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_OR_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $vgpr16 = COPY renamable $vgpr22, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_OR_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $vgpr12 = COPY renamable $vgpr18, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.60
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.57:
; GFX90A-NEXT: successors: %bb.7(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr15 = S_MOV_B32 0
- ; GFX90A-NEXT: renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_MOV_B64 0
@@ -802,9 +803,9 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr54_vgpr55 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr6_vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr62_vgpr63 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr60_vgpr61 = IMPLICIT_DEF
@@ -814,7 +815,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr42_vgpr43 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr40_vgpr41 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr46_vgpr47 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = COPY renamable $vgpr5, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr20 = COPY renamable $vgpr5, implicit $exec
; GFX90A-NEXT: renamable $sgpr23 = S_MOV_B32 0
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 0
; GFX90A-NEXT: S_BRANCH %bb.7
@@ -824,62 +825,61 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr28_vgpr29 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr26_vgpr27 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr21, implicit $exec
; GFX90A-NEXT: renamable $vgpr2_vgpr3 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
; GFX90A-NEXT: renamable $vgpr0 = COPY killed renamable $sgpr15, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.420, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.420, addrspace 3)
; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr32_vgpr33 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr26_vgpr27 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_MOV_B64 -1
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.59.bb85:
; GFX90A-NEXT: successors: %bb.56(0x40000000), %bb.60(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr12 = V_OR_B32_e32 1, $vgpr10, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr13 = COPY renamable $vgpr11, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr3 = FLAT_LOAD_UBYTE renamable $vgpr12_vgpr13, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i86)
+ ; GFX90A-NEXT: renamable $vgpr8 = V_OR_B32_e32 1, $vgpr8, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = FLAT_LOAD_UBYTE renamable $vgpr8_vgpr9, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i86)
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr3, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = COPY renamable $sgpr36_sgpr37
; GFX90A-NEXT: renamable $sgpr15 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr24 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr20 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $sgpr23 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
+ ; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr12 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr54_sgpr55 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.56, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.60.Flow31:
; GFX90A-NEXT: successors: %bb.61(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr54_sgpr55, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.61.Flow30:
; GFX90A-NEXT: successors: %bb.55(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x0000000000000003, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_XOR_B64 $exec, -1, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_OR_B64 killed renamable $sgpr52_sgpr53, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.55
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.62.bb140:
; GFX90A-NEXT: successors: %bb.68(0x40000000), %bb.63(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr30_sgpr31, implicit-def dead $scc
@@ -887,7 +887,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.63.Flow13:
; GFX90A-NEXT: successors: %bb.64(0x40000000), %bb.66(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: dead renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr24_sgpr25, $exec, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr12 = S_CSELECT_B32 1, 0, implicit killed $scc
@@ -896,68 +896,67 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.64.bb159:
; GFX90A-NEXT: successors: %bb.67(0x40000000), %bb.65(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vcc = V_CMP_NE_U32_e64 0, killed $vgpr6, implicit $exec
+ ; GFX90A-NEXT: renamable $vcc = V_CMP_NE_U32_e64 0, killed $vgpr30, implicit $exec
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.67, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.65.Flow10:
; GFX90A-NEXT: successors: %bb.66(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $sgpr12_sgpr13 = S_ANDN2_SAVEEXEC_B64 $sgpr12_sgpr13, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.66.Flow14:
; GFX90A-NEXT: successors: %bb.8(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = COPY $exec
; GFX90A-NEXT: S_BRANCH %bb.8
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.67.bb161:
; GFX90A-NEXT: successors: %bb.65(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr27, killed $vgpr29, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr33, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr17, killed $vgpr3, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr23, killed $vgpr25, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr27, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr13, killed $vgpr3, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
; GFX90A-NEXT: dead renamable $sgpr20 = S_AND_B32 killed renamable $sgpr23, 255, implicit-def $scc
; GFX90A-NEXT: renamable $vcc = S_CSELECT_B64 0, -1, implicit killed $scc
; GFX90A-NEXT: renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr18, killed $vgpr21, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr14, killed $vgpr17, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
; GFX90A-NEXT: dead renamable $sgpr15 = S_AND_B32 killed renamable $sgpr15, 255, implicit-def $scc
; GFX90A-NEXT: renamable $vcc = S_CSELECT_B64 0, -1, implicit killed $scc
; GFX90A-NEXT: renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr14, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr10, implicit $exec
; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: DS_WRITE2_B32_gfx9 killed renamable $vgpr3, killed renamable $vgpr2, renamable $vgpr3, 0, 1, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, align 4, addrspace 3)
; GFX90A-NEXT: S_BRANCH %bb.65
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.68.bb174:
; GFX90A-NEXT: successors: %bb.72(0x40000000), %bb.69(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
- ; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr30 = lr-split COPY killed renamable $vgpr14, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr34 = V_OR_B32_e32 1, $vgpr32, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr54 = V_OR_B32_e32 $vgpr34, $vgpr28, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr48 = V_OR_B32_e32 $vgpr54, $vgpr26, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr36 = V_CNDMASK_B32_e64 0, $vgpr48, 0, 0, $sgpr12_sgpr13, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr52 = V_OR_B32_e32 $vgpr36, $vgpr2, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr50 = V_OR_B32_e32 $vgpr52, $vgpr16, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr38 = V_OR_B32_e32 $vgpr50, $vgpr20, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr14 = V_CNDMASK_B32_e64 0, 0, 0, $vgpr38, killed $sgpr12_sgpr13, implicit $exec
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: renamable $vgpr28 = V_OR_B32_e32 1, $vgpr26, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr50 = V_OR_B32_e32 $vgpr28, $vgpr24, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr36 = V_OR_B32_e32 $vgpr50, $vgpr22, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr32 = V_CNDMASK_B32_e64 0, $vgpr36, 0, 0, $sgpr12_sgpr13, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr48 = V_OR_B32_e32 $vgpr32, $vgpr2, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr38 = V_OR_B32_e32 $vgpr48, $vgpr12, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr34 = V_OR_B32_e32 $vgpr38, $vgpr16, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr52 = V_CNDMASK_B32_e64 0, 0, 0, $vgpr34, killed $sgpr12_sgpr13, implicit $exec
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr28_sgpr29, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.72, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.69.Flow:
; GFX90A-NEXT: successors: %bb.70(0x40000000), %bb.71(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr30_vgpr31:0x0000000000000003, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: dead renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr12_sgpr13, $exec, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr12 = S_CSELECT_B32 1, 0, implicit killed $scc
@@ -966,49 +965,48 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.70.bb186:
; GFX90A-NEXT: successors: %bb.71(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr30_vgpr31:0x0000000000000003, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr4_vgpr5 = nsw V_LSHLREV_B64_e64 3, killed $vgpr4_vgpr5, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr27, implicit $exec
; GFX90A-NEXT: renamable $vgpr4, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr4, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr2, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr2, killed $vgpr5, killed $vcc, 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr35 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr55 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr49 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr53 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr51 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr37 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr15 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr39 = COPY renamable $vgpr35, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, renamable $vgpr34_vgpr35, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr29 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr51 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr37 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr49 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr39 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr33 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr53 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr35 = COPY renamable $vgpr29, implicit $exec
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr29, renamable $vgpr28_vgpr29, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $vgpr5 = COPY renamable $sgpr21, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr7 = COPY killed renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr7, killed renamable $vgpr48_vgpr49, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, killed renamable $vgpr52_vgpr53, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr50_vgpr51, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, killed renamable $vgpr36_vgpr37, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr5, killed renamable $vgpr14_vgpr15, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr35, killed renamable $vgpr38_vgpr39, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr11 = COPY killed renamable $sgpr22, implicit $exec
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr11, killed renamable $vgpr36_vgpr37, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr29, killed renamable $vgpr48_vgpr49, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr38_vgpr39, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr29, killed renamable $vgpr32_vgpr33, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr5, killed renamable $vgpr52_vgpr53, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr29, killed renamable $vgpr34_vgpr35, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.71.Flow9:
; GFX90A-NEXT: successors: %bb.63(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr30_vgpr31:0x0000000000000003, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vgpr14 = lr-split COPY killed renamable $vgpr30, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.63
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.72.bb196:
; GFX90A-NEXT: successors: %bb.69(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr30_vgpr31:0x0000000000000003, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x0000000000000003, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x0000000000000003, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 $vgpr14, killed $vgpr24, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr22 = V_OR_B32_e32 killed $vgpr2, killed $vgpr22, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr23, renamable $vgpr22_vgpr23, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 $vgpr52, killed $vgpr20, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr18 = V_OR_B32_e32 killed $vgpr2, killed $vgpr18, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr19, renamable $vgpr18_vgpr19, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 0
; GFX90A-NEXT: S_BRANCH %bb.69
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index 70106cf0838cd..d1578e5f283bb 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -238,18 +238,18 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
+; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
-; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
+; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
; SDAG-GFX942-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -266,10 +266,10 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x2000
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -313,23 +313,23 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX1100-NEXT: s_load_b128 s[8:11], s[4:5], 0x44
; SDAG-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x34
; SDAG-GFX1100-NEXT: s_load_b32 s15, s[4:5], 0x54
+; SDAG-GFX1100-NEXT: s_mov_b32 s16, 0
; SDAG-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s4, s1
; SDAG-GFX1100-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s12, s9
; SDAG-GFX1100-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX1100-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
; SDAG-GFX1100-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s2
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX1100-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s2
-; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s1, 0x2000
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s1
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX1100-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s1
+; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s16, 0x2000
; SDAG-GFX1100-NEXT: s_clause 0xf
; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v60, s[4:7], 0 offen
; SDAG-GFX1100-NEXT: buffer_load_b128 v[4:7], v60, s[4:7], 0 offen offset:16
@@ -386,25 +386,25 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-LABEL: memcpy_known:
; GISEL-GFX942: ; %bb.0:
; GISEL-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
-; GISEL-GFX942-NEXT: s_load_dword s11, s[4:5], 0x34
-; GISEL-GFX942-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; GISEL-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
; GISEL-GFX942-NEXT: s_mov_b32 s16, 0
; GISEL-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX942-NEXT: s_mov_b32 s8, s1
-; GISEL-GFX942-NEXT: s_mov_b32 s9, s2
-; GISEL-GFX942-NEXT: s_mov_b32 s10, s3
-; GISEL-GFX942-NEXT: s_mov_b32 s4, s13
-; GISEL-GFX942-NEXT: s_mov_b32 s5, s14
-; GISEL-GFX942-NEXT: s_mov_b32 s6, s15
+; GISEL-GFX942-NEXT: s_mov_b32 s12, s9
+; GISEL-GFX942-NEXT: s_mov_b32 s4, s1
+; GISEL-GFX942-NEXT: s_mov_b32 s5, s2
+; GISEL-GFX942-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX942-NEXT: s_mov_b32 s13, s10
+; GISEL-GFX942-NEXT: s_mov_b32 s14, s11
; GISEL-GFX942-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
-; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: s_add_i32 s2, s8, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
@@ -413,55 +413,55 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v12 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:48
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[8:11], 0 offen offset:64
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[8:11], 0 offen offset:80
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[8:11], 0 offen offset:96
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[8:11], 0 offen offset:112
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[8:11], 0 offen offset:128
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[8:11], 0 offen offset:144
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[8:11], 0 offen offset:160
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[8:11], 0 offen offset:176
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[8:11], 0 offen offset:192
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[8:11], 0 offen offset:208
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[4:7], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[4:7], 0 offen offset:240
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[12:15], 0 offen
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[12:15], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen offset:32
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[12:15], 0 offen offset:48
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[12:15], 0 offen offset:64
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[12:15], 0 offen offset:80
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[12:15], 0 offen offset:96
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[12:15], 0 offen offset:112
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[12:15], 0 offen offset:128
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[12:15], 0 offen offset:144
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[12:15], 0 offen offset:160
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[12:15], 0 offen offset:176
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[12:15], 0 offen offset:192
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[12:15], 0 offen offset:208
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[4:7], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[12:15], 0 offen offset:224
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[4:7], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[12:15], 0 offen offset:240
; GISEL-GFX942-NEXT: s_cbranch_scc1 .LBB0_1
; GISEL-GFX942-NEXT: ; %bb.2: ; %static-memcpy-post-expansion
; GISEL-GFX942-NEXT: s_endpgm
@@ -759,18 +759,18 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
+; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
-; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
+; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
; SDAG-GFX942-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -787,10 +787,10 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x100
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -834,23 +834,23 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX1100-NEXT: s_load_b128 s[8:11], s[4:5], 0x44
; SDAG-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x34
; SDAG-GFX1100-NEXT: s_load_b32 s15, s[4:5], 0x54
+; SDAG-GFX1100-NEXT: s_mov_b32 s16, 0
; SDAG-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s4, s1
; SDAG-GFX1100-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s12, s9
; SDAG-GFX1100-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX1100-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
; SDAG-GFX1100-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s2
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX1100-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s2
-; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s1, 0x100
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s1
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX1100-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s1
+; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s16, 0x100
; SDAG-GFX1100-NEXT: s_clause 0xf
; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v60, s[4:7], 0 offen
; SDAG-GFX1100-NEXT: buffer_load_b128 v[4:7], v60, s[4:7], 0 offen offset:16
@@ -907,25 +907,25 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-LABEL: memcpy_known_medium:
; GISEL-GFX942: ; %bb.0:
; GISEL-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
-; GISEL-GFX942-NEXT: s_load_dword s11, s[4:5], 0x34
-; GISEL-GFX942-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; GISEL-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
; GISEL-GFX942-NEXT: s_mov_b32 s16, 0
; GISEL-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX942-NEXT: s_mov_b32 s8, s1
-; GISEL-GFX942-NEXT: s_mov_b32 s9, s2
-; GISEL-GFX942-NEXT: s_mov_b32 s10, s3
-; GISEL-GFX942-NEXT: s_mov_b32 s4, s13
-; GISEL-GFX942-NEXT: s_mov_b32 s5, s14
-; GISEL-GFX942-NEXT: s_mov_b32 s6, s15
+; GISEL-GFX942-NEXT: s_mov_b32 s12, s9
+; GISEL-GFX942-NEXT: s_mov_b32 s4, s1
+; GISEL-GFX942-NEXT: s_mov_b32 s5, s2
+; GISEL-GFX942-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX942-NEXT: s_mov_b32 s13, s10
+; GISEL-GFX942-NEXT: s_mov_b32 s14, s11
; GISEL-GFX942-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
-; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: s_add_i32 s2, s8, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
@@ -934,55 +934,55 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v12 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:48
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[8:11], 0 offen offset:64
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[8:11], 0 offen offset:80
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[8:11], 0 offen offset:96
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[8:11], 0 offen offset:112
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[8:11], 0 offen offset:128
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[8:11], 0 offen offset:144
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[8:11], 0 offen offset:160
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[8:11], 0 offen offset:176
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[8:11], 0 offen offset:192
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[8:11], 0 offen offset:208
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[4:7], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[4:7], 0 offen offset:240
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[12:15], 0 offen
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[12:15], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen offset:32
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[12:15], 0 offen offset:48
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[12:15], 0 offen offset:64
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[12:15], 0 offen offset:80
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[12:15], 0 offen offset:96
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[12:15], 0 offen offset:112
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[12:15], 0 offen offset:128
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[12:15], 0 offen offset:144
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[12:15], 0 offen offset:160
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[12:15], 0 offen offset:176
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[12:15], 0 offen offset:192
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[12:15], 0 offen offset:208
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[4:7], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[12:15], 0 offen offset:224
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[4:7], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[12:15], 0 offen offset:240
; GISEL-GFX942-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-GFX942-NEXT: ; %bb.2: ; %static-memcpy-post-expansion
; GISEL-GFX942-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
index 4010766d2add3..c4be23eb40c95 100644
--- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll
+++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
@@ -8,147 +8,151 @@ define i64 @sdiv64(i64 %a, i64 %b) {
; GFX9-LABEL: sdiv64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v3
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB0_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
-; GFX9-NEXT: v_xor_b32_e32 v8, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v9, v2, v7
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v9
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v8
-; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v9
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v8, vcc
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v2
-; GFX9-NEXT: v_mul_lo_u32 v5, v12, v10
-; GFX9-NEXT: v_mul_lo_u32 v4, v13, v11
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; GFX9-NEXT: v_add3_u32 v14, v3, v5, v4
-; GFX9-NEXT: v_mul_hi_u32 v15, v11, v2
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v14, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v15, vcc, v15, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v14, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v10, v3, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; GFX9-NEXT: v_mul_lo_u32 v4, v12, v10
-; GFX9-NEXT: v_mul_lo_u32 v5, v13, v11
-; GFX9-NEXT: v_mul_hi_u32 v13, v11, v2
-; GFX9-NEXT: v_add3_u32 v12, v3, v4, v5
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v12, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v12, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v13, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v11, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v10, v3, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v5
-; GFX9-NEXT: v_xor_b32_e32 v6, v0, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; GFX9-NEXT: v_mul_hi_u32 v10, v6, v2
-; GFX9-NEXT: v_xor_b32_e32 v11, v3, v5
+; GFX9-NEXT: s_cbranch_execnz .LBB0_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT: s_cbranch_execnz .LBB0_4
+; GFX9-NEXT: .LBB0_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB0_3:
+; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v8, vcc
+; GFX9-NEXT: v_xor_b32_e32 v9, v1, v8
+; GFX9-NEXT: v_xor_b32_e32 v10, v0, v8
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v10
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v9
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v10
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v9, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v3
+; GFX9-NEXT: v_mul_lo_u32 v2, v13, v11
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_add3_u32 v14, v1, v6, v2
+; GFX9-NEXT: v_mul_hi_u32 v15, v11, v0
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v14, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v15, vcc, v15, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v14, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v15, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v12, v3
+; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
+; GFX9-NEXT: v_mul_hi_u32 v13, v11, v0
+; GFX9-NEXT: v_add3_u32 v12, v1, v2, v6
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v12, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v12, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v13, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v3, v1, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v7
+; GFX9-NEXT: v_xor_b32_e32 v4, v0, v7
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v6, 0
+; GFX9-NEXT: v_mul_hi_u32 v5, v4, v2
+; GFX9-NEXT: v_xor_b32_e32 v11, v3, v7
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v0
+; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v0
; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v5, v2
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v12, v3, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mul_lo_u32 v4, v8, v2
-; GFX9-NEXT: v_mul_lo_u32 v10, v9, v3
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v2, 0
-; GFX9-NEXT: v_add3_u32 v1, v1, v10, v4
-; GFX9-NEXT: v_sub_u32_e32 v4, v11, v1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v6, v0
-; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v8, vcc
-; GFX9-NEXT: v_sub_co_u32_e64 v6, s[4:5], v0, v9
+; GFX9-NEXT: v_mul_lo_u32 v5, v9, v2
+; GFX9-NEXT: v_mul_lo_u32 v6, v10, v3
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v2, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
+; GFX9-NEXT: v_sub_u32_e32 v5, v11, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v4, v0
+; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v5, v9, vcc
+; GFX9-NEXT: v_sub_co_u32_e64 v5, s[4:5], v0, v10
; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v6, v9
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v9
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[4:5]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v10, v6, s[4:5]
-; GFX9-NEXT: v_add_co_u32_e64 v6, s[4:5], 2, v2
-; GFX9-NEXT: v_addc_co_u32_e64 v10, s[4:5], 0, v3, s[4:5]
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v5, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v5, s[4:5]
+; GFX9-NEXT: v_add_co_u32_e64 v5, s[4:5], 2, v2
+; GFX9-NEXT: v_addc_co_u32_e64 v6, s[4:5], 0, v3, s[4:5]
; GFX9-NEXT: v_add_co_u32_e64 v12, s[4:5], 1, v2
; GFX9-NEXT: v_addc_co_u32_e64 v13, s[4:5], 0, v3, s[4:5]
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v11, v1, vcc
; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v13, v10, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v13, v6, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10
; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v12, v6, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-NEXT: v_xor_b32_e32 v2, v5, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v12, v5, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v2
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v1, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v0, v2, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB0_2: ; %Flow
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT: v_xor_b32_e32 v2, v7, v8
+; GFX9-NEXT: v_xor_b32_e32 v3, v0, v2
+; GFX9-NEXT: v_xor_b32_e32 v0, v1, v2
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v2, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr2
+; GFX9-NEXT: ; implicit-def: $vgpr4
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; GFX9-NEXT: s_cbranch_execz .LBB0_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB0_2
+; GFX9-NEXT: .LBB0_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v2
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v4, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_sub_u32_e32 v1, v4, v1
+; GFX9-NEXT: v_sub_u32_e32 v4, v1, v2
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v1
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v3, vcc
-; GFX9-NEXT: .LBB0_4: ; %.split
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = sdiv i64 %a, %b
ret i64 %d
@@ -158,76 +162,84 @@ define i64 @udiv64(i64 %a, i64 %b) {
; GFX9-LABEL: udiv64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v3
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB1_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v13, vcc, 0, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v3, vcc
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; GFX9-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v4
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v12
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; GFX9-NEXT: v_add3_u32 v9, v5, v6, v7
-; GFX9-NEXT: v_mul_hi_u32 v15, v12, v4
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v9, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v9, 0
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v5, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v12, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v11, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v9, 0
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v10
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v9
-; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4
-; GFX9-NEXT: v_add3_u32 v11, v5, v6, v7
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v11, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v11, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v12, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v10, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v8, 0
-; GFX9-NEXT: v_mul_hi_u32 v9, v0, v6
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v6, 0
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v8, 0
+; GFX9-NEXT: s_cbranch_execnz .LBB1_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT: s_cbranch_execnz .LBB1_4
+; GFX9-NEXT: .LBB1_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB1_3:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v3
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v3, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v10
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v11
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_add3_u32 v8, v1, v6, v7
+; GFX9-NEXT: v_mul_hi_u32 v14, v11, v0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v11, v8, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v10, v8, 0
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v14, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v7, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v9
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v8
+; GFX9-NEXT: v_mul_hi_u32 v11, v8, v0
+; GFX9-NEXT: v_add3_u32 v10, v1, v6, v7
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v10, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, 0, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v9, v10, 0
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v12, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v8, 0
+; GFX9-NEXT: v_mul_hi_u32 v9, v4, v6
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v8, 0
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v6
; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v10, v7, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v5, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
; GFX9-NEXT: v_mul_lo_u32 v8, v3, v6
; GFX9-NEXT: v_mul_lo_u32 v9, v2, v7
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0
-; GFX9-NEXT: v_add3_u32 v5, v5, v9, v8
-; GFX9-NEXT: v_sub_u32_e32 v8, v1, v5
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, v6, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v9, v8
+; GFX9-NEXT: v_sub_u32_e32 v8, v5, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v4, v0
; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v8, v3, vcc
; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v0, v2
; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5]
@@ -238,7 +250,7 @@ define i64 @udiv64(i64 %a, i64 %b) {
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v3
; GFX9-NEXT: v_cndmask_b32_e64 v4, v9, v8, s[4:5]
; GFX9-NEXT: v_add_co_u32_e64 v8, s[4:5], 2, v6
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v5, v1, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, v7, s[4:5]
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
; GFX9-NEXT: v_add_co_u32_e64 v10, s[4:5], 1, v6
@@ -252,38 +264,34 @@ define i64 @udiv64(i64 %a, i64 %b) {
; GFX9-NEXT: v_cndmask_b32_e64 v4, v11, v9, s[4:5]
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: v_cndmask_b32_e64 v0, v10, v8, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v0, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB1_2: ; %Flow
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr2
+; GFX9-NEXT: ; implicit-def: $vgpr4
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; GFX9-NEXT: s_cbranch_execz .LBB1_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB1_2
+; GFX9-NEXT: .LBB1_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v2
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v4, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_sub_u32_e32 v1, v4, v1
+; GFX9-NEXT: v_sub_u32_e32 v4, v1, v2
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_add_u32_e32 v3, 1, v1
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v3, vcc
-; GFX9-NEXT: .LBB1_4: ; %.split
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = udiv i64 %a, %b
ret i64 %d
@@ -293,143 +301,147 @@ define i64 @srem64(i64 %a, i64 %b) {
; GFX9-LABEL: srem64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v3
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB2_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GFX9-NEXT: v_xor_b32_e32 v7, v3, v4
-; GFX9-NEXT: v_xor_b32_e32 v8, v2, v4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v8
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v7
-; GFX9-NEXT: v_sub_co_u32_e32 v11, vcc, 0, v8
-; GFX9-NEXT: v_subb_co_u32_e32 v12, vcc, 0, v7, vcc
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v9, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v2
-; GFX9-NEXT: v_mul_lo_u32 v5, v11, v9
-; GFX9-NEXT: v_mul_lo_u32 v4, v12, v10
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; GFX9-NEXT: v_add3_u32 v13, v3, v5, v4
-; GFX9-NEXT: v_mul_hi_u32 v14, v10, v2
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v13, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v14, vcc, v14, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v13, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v14, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v9, v3, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v10, 0
-; GFX9-NEXT: v_mul_lo_u32 v4, v11, v9
-; GFX9-NEXT: v_mul_lo_u32 v5, v12, v10
-; GFX9-NEXT: v_mul_hi_u32 v12, v10, v2
-; GFX9-NEXT: v_add3_u32 v11, v3, v4, v5
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v10, v11, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v11, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v12, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v9, v3, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v5
-; GFX9-NEXT: v_xor_b32_e32 v6, v0, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
-; GFX9-NEXT: v_mul_hi_u32 v9, v6, v2
-; GFX9-NEXT: v_xor_b32_e32 v10, v3, v5
+; GFX9-NEXT: s_cbranch_execnz .LBB2_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB2_4
+; GFX9-NEXT: .LBB2_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB2_3:
+; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT: v_add_co_u32_e32 v1, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v3, v0, vcc
+; GFX9-NEXT: v_xor_b32_e32 v8, v2, v0
+; GFX9-NEXT: v_xor_b32_e32 v9, v1, v0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v9
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v8
+; GFX9-NEXT: v_sub_co_u32_e32 v11, vcc, 0, v9
+; GFX9-NEXT: v_subb_co_u32_e32 v12, vcc, 0, v8, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v11, v3
+; GFX9-NEXT: v_mul_lo_u32 v2, v12, v10
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v10, 0
+; GFX9-NEXT: v_add3_u32 v13, v1, v6, v2
+; GFX9-NEXT: v_mul_hi_u32 v14, v10, v0
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v13, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v14, vcc, v14, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v13, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v14, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v10, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v10, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v11, v3
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v10
+; GFX9-NEXT: v_mul_hi_u32 v12, v10, v0
+; GFX9-NEXT: v_add3_u32 v11, v1, v2, v6
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v11, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v11, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v12, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v3, v1, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v7
+; GFX9-NEXT: v_xor_b32_e32 v4, v0, v7
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v6, 0
+; GFX9-NEXT: v_mul_hi_u32 v5, v4, v2
+; GFX9-NEXT: v_xor_b32_e32 v10, v3, v7
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v0
+; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v0
; GFX9-NEXT: v_addc_co_u32_e32 v11, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v9, v2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v5, v2
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v11, v3, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mul_lo_u32 v2, v7, v0
-; GFX9-NEXT: v_mul_lo_u32 v3, v8, v1
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v0, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v8, v0
+; GFX9-NEXT: v_mul_lo_u32 v3, v9, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v0, 0
; GFX9-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-NEXT: v_sub_u32_e32 v2, v10, v1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v6, v0
-; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v7, vcc
-; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v0, v8
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v4, v0
+; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v8, vcc
+; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v0, v9
; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[6:7], 0, v2, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v7
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v4, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v9
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[6:7]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v3, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v7
-; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v7, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[6:7]
-; GFX9-NEXT: v_sub_co_u32_e64 v9, s[4:5], v3, v8
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v4, v8
+; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v2, v8, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[6:7]
+; GFX9-NEXT: v_sub_co_u32_e64 v6, s[4:5], v3, v9
; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v10, v1, vcc
-; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v6
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v7
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v5
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, v2, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v9, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v6, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v5
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v0, v5
-; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v1, v5, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB2_2: ; %Flow
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v7
+; GFX9-NEXT: v_xor_b32_e32 v1, v1, v7
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v7
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr2
+; GFX9-NEXT: ; implicit-def: $vgpr4
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_cbranch_execz .LBB2_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB2_2
+; GFX9-NEXT: .LBB2_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v2
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v4, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, v4, v0
; GFX9-NEXT: v_sub_u32_e32 v1, v0, v2
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX9-NEXT: v_sub_u32_e32 v1, v0, v2
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; GFX9-NEXT: .LBB2_4: ; %.split
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = srem i64 %a, %b
ret i64 %d
@@ -439,83 +451,91 @@ define i64 @urem64(i64 %a, i64 %b) {
; GFX9-LABEL: urem64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v3
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB3_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v13, vcc, 0, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v3, vcc
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; GFX9-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v4
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v12
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; GFX9-NEXT: v_add3_u32 v9, v5, v6, v7
-; GFX9-NEXT: v_mul_hi_u32 v15, v12, v4
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v9, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v9, 0
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v5, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v12, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v11, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v9, 0
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v10
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v9
-; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4
-; GFX9-NEXT: v_add3_u32 v11, v5, v6, v7
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v11, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v11, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v12, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v10, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v8, 0
-; GFX9-NEXT: v_mul_hi_u32 v9, v0, v6
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v6, 0
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v8, 0
+; GFX9-NEXT: s_cbranch_execnz .LBB3_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB3_4
+; GFX9-NEXT: .LBB3_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB3_3:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v3
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v3, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v10
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v11
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_add3_u32 v8, v1, v6, v7
+; GFX9-NEXT: v_mul_hi_u32 v14, v11, v0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v11, v8, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v10, v8, 0
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v14, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v7, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v9
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v8
+; GFX9-NEXT: v_mul_hi_u32 v11, v8, v0
+; GFX9-NEXT: v_add3_u32 v10, v1, v6, v7
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v10, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, 0, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v9, v10, 0
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v12, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v8, 0
+; GFX9-NEXT: v_mul_hi_u32 v9, v4, v6
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v8, 0
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v6
; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v10, v7, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mul_lo_u32 v6, v3, v4
-; GFX9-NEXT: v_mul_lo_u32 v7, v2, v5
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v4, 0
-; GFX9-NEXT: v_add3_u32 v5, v5, v7, v6
-; GFX9-NEXT: v_sub_u32_e32 v6, v1, v5
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mul_lo_u32 v6, v3, v0
+; GFX9-NEXT: v_mul_lo_u32 v7, v2, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, v0, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v7, v6
+; GFX9-NEXT: v_sub_u32_e32 v6, v5, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v4, v0
; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v6, v3, vcc
; GFX9-NEXT: v_sub_co_u32_e64 v6, s[4:5], v0, v2
; GFX9-NEXT: v_subbrev_co_u32_e64 v7, s[6:7], 0, v4, s[4:5]
; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v7, v3
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[6:7]
; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v5, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[6:7]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v7, v3
; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v4, v3, s[4:5]
@@ -531,37 +551,33 @@ define i64 @urem64(i64 %a, i64 %b) {
; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[4:5]
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v1, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, v9, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB3_2: ; %Flow
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v9, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr2
+; GFX9-NEXT: ; implicit-def: $vgpr4
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_cbranch_execz .LBB3_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB3_2
+; GFX9-NEXT: .LBB3_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v2
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v4, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2
+; GFX9-NEXT: v_sub_u32_e32 v0, v4, v0
; GFX9-NEXT: v_sub_u32_e32 v1, v0, v2
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX9-NEXT: v_sub_u32_e32 v1, v0, v2
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc
-; GFX9-NEXT: .LBB3_4: ; %.split
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = urem i64 %a, %b
ret i64 %d
@@ -694,164 +710,168 @@ define <2 x i64> @sdivrem64(i64 %a, i64 %b) {
; GFX9-LABEL: sdivrem64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v7
+; GFX9-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB8_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
-; GFX9-NEXT: v_xor_b32_e32 v8, v3, v7
-; GFX9-NEXT: v_xor_b32_e32 v9, v2, v7
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v9
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v8
-; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v9
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v8, vcc
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0x4f800000, v2
-; GFX9-NEXT: v_rcp_f32_e32 v2, v2
-; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
-; GFX9-NEXT: v_trunc_f32_e32 v3, v3
-; GFX9-NEXT: v_madmk_f32 v2, v3, 0xcf800000, v2
-; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v3
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v2
-; GFX9-NEXT: v_mul_lo_u32 v5, v12, v10
-; GFX9-NEXT: v_mul_lo_u32 v4, v13, v11
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; GFX9-NEXT: v_add3_u32 v14, v3, v5, v4
-; GFX9-NEXT: v_mul_hi_u32 v15, v11, v2
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v14, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v15, vcc, v15, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v14, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v10, v3, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v11, 0
-; GFX9-NEXT: v_mul_lo_u32 v4, v12, v10
-; GFX9-NEXT: v_mul_lo_u32 v5, v13, v11
-; GFX9-NEXT: v_mul_hi_u32 v13, v11, v2
-; GFX9-NEXT: v_add3_u32 v12, v3, v4, v5
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v12, 0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v10, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v3
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v12, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v13, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v11, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v10, v3, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v10, 31, v1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v10
-; GFX9-NEXT: v_xor_b32_e32 v5, v0, v10
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v10, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v4, 0
-; GFX9-NEXT: v_mul_hi_u32 v6, v5, v2
-; GFX9-NEXT: v_xor_b32_e32 v11, v3, v10
+; GFX9-NEXT: s_cbranch_execnz .LBB8_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB8_4
+; GFX9-NEXT: .LBB8_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB8_3:
+; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v7, v8, vcc
+; GFX9-NEXT: v_xor_b32_e32 v9, v1, v8
+; GFX9-NEXT: v_xor_b32_e32 v10, v0, v8
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v10
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v9
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v10
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v9, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v3
+; GFX9-NEXT: v_mul_lo_u32 v2, v13, v11
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_add3_u32 v14, v1, v6, v2
+; GFX9-NEXT: v_mul_hi_u32 v15, v11, v0
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v14, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v15, vcc, v15, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v14, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v15, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v12, v3
+; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
+; GFX9-NEXT: v_mul_hi_u32 v13, v11, v0
+; GFX9-NEXT: v_add3_u32 v12, v1, v2, v6
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v12, 0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v13, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v12, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v13, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v7, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v3, v1, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v7
+; GFX9-NEXT: v_xor_b32_e32 v4, v0, v7
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v5, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v6, 0
+; GFX9-NEXT: v_mul_hi_u32 v5, v4, v2
+; GFX9-NEXT: v_xor_b32_e32 v11, v3, v7
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v2, 0
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v0
+; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v5, v0
; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v5, v2
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v12, v3, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
-; GFX9-NEXT: v_mul_lo_u32 v4, v8, v2
-; GFX9-NEXT: v_mul_lo_u32 v6, v9, v3
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v2, 0
-; GFX9-NEXT: v_add3_u32 v1, v1, v6, v4
-; GFX9-NEXT: v_sub_u32_e32 v4, v11, v1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v5, v0
-; GFX9-NEXT: v_subb_co_u32_e64 v6, s[4:5], v4, v8, vcc
-; GFX9-NEXT: v_sub_co_u32_e64 v12, s[4:5], v0, v9
-; GFX9-NEXT: v_subbrev_co_u32_e64 v13, s[6:7], 0, v6, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v13, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[6:7]
+; GFX9-NEXT: v_mul_lo_u32 v5, v9, v2
+; GFX9-NEXT: v_mul_lo_u32 v6, v10, v3
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v2, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5
+; GFX9-NEXT: v_sub_u32_e32 v5, v11, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v0
+; GFX9-NEXT: v_subb_co_u32_e64 v5, s[4:5], v5, v9, vcc
+; GFX9-NEXT: v_sub_co_u32_e64 v6, s[4:5], v4, v10
+; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[6:7], 0, v5, s[4:5]
; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v12, v9
-; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[6:7]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v13, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
-; GFX9-NEXT: v_add_co_u32_e64 v5, s[6:7], 2, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[6:7]
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v6, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[6:7]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v12, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v13, s[6:7]
+; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 2, v2
; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v3, s[6:7]
; GFX9-NEXT: v_add_co_u32_e64 v15, s[6:7], 1, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v11, v1, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v1, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v16, s[6:7], 0, v3, s[6:7]
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v8
-; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
-; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v9
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v16, v14, s[6:7]
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v11, v9
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v4, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v16, v14, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, -1, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v8
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v11, v14, vcc
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v15, v5, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_xor_b32_e32 v5, v10, v7
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v5
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v5
-; GFX9-NEXT: v_sub_co_u32_e64 v4, s[8:9], v2, v5
-; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v6, v8, s[4:5]
-; GFX9-NEXT: v_subb_co_u32_e64 v5, s[8:9], v3, v5, s[8:9]
-; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v12, v9
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v11, v9
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v14, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v15, v13, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT: v_xor_b32_e32 v2, v7, v8
+; GFX9-NEXT: v_xor_b32_e32 v3, v0, v2
+; GFX9-NEXT: v_xor_b32_e32 v0, v1, v2
+; GFX9-NEXT: v_sub_co_u32_e64 v0, s[8:9], v0, v2
+; GFX9-NEXT: v_subb_co_u32_e64 v1, s[8:9], v3, v2, s[8:9]
+; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v5, v9, s[4:5]
+; GFX9-NEXT: v_sub_co_u32_e64 v3, s[4:5], v6, v10
; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v13, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v12, v3, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v10
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v10
-; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v0, v10
-; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v1, v10, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB8_2: ; %Flow
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v12, v2, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX9-NEXT: v_xor_b32_e32 v4, v2, v7
+; GFX9-NEXT: v_xor_b32_e32 v2, v3, v7
+; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v7
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v7, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr6
+; GFX9-NEXT: ; implicit-def: $vgpr4
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
-; GFX9-NEXT: s_cbranch_execz .LBB8_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB8_2
+; GFX9-NEXT: .LBB8_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v6
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v6
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v4, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v0, v6
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_sub_u32_e32 v2, v4, v2
+; GFX9-NEXT: v_sub_u32_e32 v4, v2, v6
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc
-; GFX9-NEXT: .LBB8_4: ; %.split
+; GFX9-NEXT: v_sub_u32_e32 v3, v2, v6
+; GFX9-NEXT: v_add_u32_e32 v4, 1, v0
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NEXT: v_mov_b32_e32 v2, v6
-; GFX9-NEXT: v_mov_b32_e32 v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = sdiv i64 %a, %b
%r = srem i64 %a, %b
@@ -864,145 +884,149 @@ define <2 x i64> @udivrem64(i64 %a, i64 %b) {
; GFX9-LABEL: udivrem64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v6, v3
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: v_or_b32_e32 v0, v7, v6
+; GFX9-NEXT: v_mov_b32_e32 v4, v2
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB9_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v13, vcc, 0, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v3, vcc
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; GFX9-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v4
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v12
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; GFX9-NEXT: v_add3_u32 v9, v5, v6, v7
-; GFX9-NEXT: v_mul_hi_u32 v15, v12, v4
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v9, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v9, 0
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v5, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v12, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v11, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v9, 0
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v10
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v9
-; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4
-; GFX9-NEXT: v_add3_u32 v11, v5, v6, v7
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v11, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v11, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v12, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v10, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v8, 0
-; GFX9-NEXT: v_mul_hi_u32 v9, v0, v6
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v6, 0
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v8, 0
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v6
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v10, v7, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mul_lo_u32 v8, v3, v6
-; GFX9-NEXT: v_mul_lo_u32 v9, v2, v7
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v6, 0
-; GFX9-NEXT: v_add3_u32 v5, v5, v9, v8
-; GFX9-NEXT: v_sub_u32_e32 v8, v1, v5
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v4
-; GFX9-NEXT: v_subb_co_u32_e64 v8, s[4:5], v8, v3, vcc
-; GFX9-NEXT: v_sub_co_u32_e64 v9, s[4:5], v0, v2
+; GFX9-NEXT: s_cbranch_execnz .LBB9_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; GFX9-NEXT: s_cbranch_execnz .LBB9_4
+; GFX9-NEXT: .LBB9_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB9_3:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v6
+; GFX9-NEXT: v_sub_co_u32_e32 v13, vcc, 0, v4
+; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v6, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v0
+; GFX9-NEXT: v_mul_lo_u32 v2, v13, v3
+; GFX9-NEXT: v_mul_lo_u32 v8, v14, v12
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v12, 0
+; GFX9-NEXT: v_add3_u32 v10, v1, v2, v8
+; GFX9-NEXT: v_mul_hi_u32 v15, v12, v0
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v12, v10, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v3, v10, 0
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v15, v1
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v1, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v10
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v12, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v10, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v13, v3
+; GFX9-NEXT: v_mul_lo_u32 v8, v14, v10
+; GFX9-NEXT: v_mul_hi_u32 v12, v10, v0
+; GFX9-NEXT: v_add3_u32 v11, v1, v2, v8
+; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v11, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v3, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v1
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v11, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v12, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v8, 0
+; GFX9-NEXT: v_mul_hi_u32 v9, v5, v2
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
+; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v8, 0
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v9, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, v10, v3, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mul_lo_u32 v8, v6, v2
+; GFX9-NEXT: v_mul_lo_u32 v9, v4, v3
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v9, v8
+; GFX9-NEXT: v_sub_u32_e32 v8, v7, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v0
+; GFX9-NEXT: v_subb_co_u32_e64 v8, s[4:5], v8, v6, vcc
+; GFX9-NEXT: v_sub_co_u32_e64 v9, s[4:5], v5, v4
; GFX9-NEXT: v_subbrev_co_u32_e64 v10, s[6:7], 0, v8, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v10, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[6:7]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v2
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v10, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[6:7]
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v9, v4
; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[6:7]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v10, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v11, s[6:7]
-; GFX9-NEXT: v_add_co_u32_e64 v11, s[6:7], 2, v6
-; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, v7, s[6:7]
-; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 1, v6
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
-; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v7, s[6:7]
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
-; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v14, v12, s[6:7]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], v10, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v11, s[6:7]
+; GFX9-NEXT: v_add_co_u32_e64 v11, s[6:7], 2, v2
+; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, v3, s[6:7]
+; GFX9-NEXT: v_add_co_u32_e64 v13, s[6:7], 1, v2
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e64 v14, s[6:7], 0, v3, s[6:7]
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v7, v6
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v5, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v14, v12, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX9-NEXT: v_subb_co_u32_e64 v3, s[4:5], v8, v3, s[4:5]
-; GFX9-NEXT: v_sub_co_u32_e64 v2, s[4:5], v9, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc
-; GFX9-NEXT: v_subbrev_co_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v10, v3, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v13, v11, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v9, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB9_2: ; %Flow
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v7, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v13, v11, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v8, v6, s[4:5]
+; GFX9-NEXT: v_sub_co_u32_e64 v4, s[4:5], v9, v4
+; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, v4, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr4
+; GFX9-NEXT: ; implicit-def: $vgpr5
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GFX9-NEXT: s_cbranch_execz .LBB9_4
-; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, 0, v2
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, v1
-; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT: s_cbranch_execz .LBB9_2
+; GFX9-NEXT: .LBB9_4:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v4
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v4
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v5, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mul_lo_u32 v2, v0, v4
+; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
+; GFX9-NEXT: v_sub_u32_e32 v2, v5, v2
+; GFX9-NEXT: v_sub_u32_e32 v5, v2, v4
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_sub_u32_e32 v3, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc
-; GFX9-NEXT: .LBB9_4: ; %.split
+; GFX9-NEXT: v_sub_u32_e32 v3, v2, v4
+; GFX9-NEXT: v_add_u32_e32 v5, 1, v0
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v4
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NEXT: v_mov_b32_e32 v2, v6
-; GFX9-NEXT: v_mov_b32_e32 v3, v7
; GFX9-NEXT: s_setpc_b64 s[30:31]
%d = udiv i64 %a, %b
%r = urem i64 %a, %b
@@ -1015,134 +1039,137 @@ define i64 @sdiv64_known32(i64 %a, i64 %b) {
; GFX9-LABEL: sdiv64_known32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX9-NEXT: v_ashrrev_i32_e32 v0, 31, v3
-; GFX9-NEXT: v_or_b32_e32 v4, v2, v0
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX9-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v2
+; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX9-NEXT: v_or_b32_e32 v0, v5, v4
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[4:5]
-; GFX9-NEXT: s_cbranch_execz .LBB10_2
-; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, v0
-; GFX9-NEXT: v_sub_co_u32_e32 v13, vcc, 0, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v0, vcc
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0x4f800000, v4
-; GFX9-NEXT: v_rcp_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_madmk_f32 v4, v5, 0xcf800000, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v5
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v4
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v11
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v12
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v12, 0
-; GFX9-NEXT: v_add3_u32 v9, v5, v6, v7
-; GFX9-NEXT: v_mul_hi_u32 v15, v12, v4
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v12, v9, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v4, 0
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v9, 0
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v15, v5
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v5, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v9
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v12, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, v11, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v9, 0
-; GFX9-NEXT: v_mul_lo_u32 v6, v13, v10
-; GFX9-NEXT: v_mul_lo_u32 v7, v14, v9
-; GFX9-NEXT: v_mul_hi_u32 v12, v9, v4
-; GFX9-NEXT: v_add3_u32 v11, v5, v6, v7
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v11, 0
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v5
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v11, 0
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v12, v7
-; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v6, v8, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v10, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v8, 0
-; GFX9-NEXT: v_mul_hi_u32 v9, v1, v6
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v2, v6, 0
-; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v2, v8, 0
+; GFX9-NEXT: s_cbranch_execnz .LBB10_3
+; GFX9-NEXT: ; %bb.1: ; %Flow
+; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT: s_cbranch_execnz .LBB10_4
+; GFX9-NEXT: .LBB10_2: ; %.split
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: .LBB10_3:
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v4
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, 0, v3
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v4, vcc
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0
+; GFX9-NEXT: v_rcp_f32_e32 v0, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
+; GFX9-NEXT: v_trunc_f32_e32 v1, v1
+; GFX9-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v10, v1
+; GFX9-NEXT: v_cvt_u32_f32_e32 v11, v0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v10
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v11
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
+; GFX9-NEXT: v_add3_u32 v8, v1, v6, v7
+; GFX9-NEXT: v_mul_hi_u32 v14, v11, v0
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v11, v8, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v0, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v10, v8, 0
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v14, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v7, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v9, vcc, v10, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v8, 0
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v9
+; GFX9-NEXT: v_mul_lo_u32 v7, v13, v8
+; GFX9-NEXT: v_mul_hi_u32 v11, v8, v0
+; GFX9-NEXT: v_add3_u32 v10, v1, v6, v7
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v10, 0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v0, 0
+; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v11, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v12, vcc, 0, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v9, v10, 0
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v11, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, v12, v1, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v2, v8, 0
+; GFX9-NEXT: v_mul_hi_u32 v9, v2, v6
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v5, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, v9, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v10, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v5, v8, 0
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v9, v6
; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v10, v7, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v5, vcc
-; GFX9-NEXT: v_mul_lo_u32 v8, v0, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v0
+; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
+; GFX9-NEXT: v_mul_lo_u32 v8, v4, v6
; GFX9-NEXT: v_mul_lo_u32 v9, v3, v7
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, v6, 0
-; GFX9-NEXT: v_add3_u32 v5, v5, v9, v8
-; GFX9-NEXT: v_sub_u32_e32 v8, v2, v5
-; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v4
-; GFX9-NEXT: v_subb_co_u32_e64 v4, s[4:5], v8, v0, vcc
-; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v1, v3
-; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[4:5], 0, v4, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v4, v0
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v6, 0
+; GFX9-NEXT: v_add3_u32 v1, v1, v9, v8
+; GFX9-NEXT: v_sub_u32_e32 v8, v5, v1
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v0
+; GFX9-NEXT: v_subb_co_u32_e64 v2, s[4:5], v8, v4, vcc
+; GFX9-NEXT: v_sub_co_u32_e64 v8, s[4:5], v0, v3
+; GFX9-NEXT: v_subbrev_co_u32_e64 v2, s[4:5], 0, v2, s[4:5]
+; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v4
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
; GFX9-NEXT: v_cmp_ge_u32_e64 s[4:5], v8, v3
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v9, v8, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], v2, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v9, v8, s[4:5]
; GFX9-NEXT: v_add_co_u32_e64 v8, s[4:5], 2, v6
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v5, v1, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, v7, s[4:5]
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v2, v0
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
; GFX9-NEXT: v_add_co_u32_e64 v10, s[4:5], 1, v6
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
+; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v0, v3
; GFX9-NEXT: v_addc_co_u32_e64 v11, s[4:5], 0, v7, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v0
-; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v11, v9, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v2
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v11, v9, s[4:5]
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: v_cndmask_b32_e64 v0, v10, v8, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v0, vcc
-; GFX9-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: .LBB10_2: ; %Flow
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX9-NEXT: ; implicit-def: $vgpr3
+; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[6:7]
-; GFX9-NEXT: s_cbranch_execz .LBB10_4
-; GFX9-NEXT: ; %bb.3:
+; GFX9-NEXT: s_cbranch_execz .LBB10_2
+; GFX9-NEXT: .LBB10_4:
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v3
-; GFX9-NEXT: v_sub_u32_e32 v2, 0, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v3
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, v0
-; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2
-; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_mul_hi_u32 v0, v1, v0
-; GFX9-NEXT: v_mul_lo_u32 v2, v0, v3
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, v0
+; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_hi_u32 v0, v2, v0
+; GFX9-NEXT: v_mul_lo_u32 v1, v0, v3
; GFX9-NEXT: v_add_u32_e32 v4, 1, v0
-; GFX9-NEXT: v_sub_u32_e32 v1, v1, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, v2, v1
; GFX9-NEXT: v_sub_u32_e32 v2, v1, v3
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GFX9-NEXT: v_add_u32_e32 v2, 1, v0
; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc
-; GFX9-NEXT: .LBB10_4: ; %.split
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a.ext = ashr i64 %a, 32
%b.ext = ashr i64 %b, 32
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 50708b4915c30..0e795cf3b3ead 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -3880,7 +3880,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB36_2
; GFX950-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private
@@ -4056,7 +4056,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB37_2
; GFX950-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private
@@ -4820,7 +4820,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB44_2
; GFX950-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private
@@ -4996,7 +4996,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB45_2
; GFX950-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private
@@ -5760,7 +5760,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB52_2
; GFX950-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private
@@ -5936,7 +5936,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-SDAG-NEXT: buffer_inv sc1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-SDAG-NEXT: ; implicit-def: $vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
; GFX950-SDAG-NEXT: s_cbranch_execz .LBB53_2
; GFX950-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 954aba0f812aa..319f552df7456 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -61,7 +61,7 @@ define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v8, v[2:3]
; SDAG-NEXT: v_add3_u32 v4, v9, v6, v7
-; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr6
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: ; implicit-def: $vgpr8
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
@@ -271,7 +271,7 @@ define i128 @fptoui_f64_to_i128(double %x) {
; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr6
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: .LBB1_3: ; %Flow
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
@@ -356,74 +356,74 @@ define i128 @fptosi_f32_to_i128(float %x) {
; SDAG: ; %bb.0: ; %fp-to-i-entry
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v4, v0
-; SDAG-NEXT: v_bfe_u32 v9, v4, 23, 8
+; SDAG-NEXT: v_bfe_u32 v8, v4, 23, 8
; SDAG-NEXT: s_movk_i32 s4, 0x7e
; SDAG-NEXT: v_mov_b32_e32 v0, 0
; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mov_b32_e32 v3, 0
-; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
+; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v8
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_cbranch_execz .LBB2_6
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v4
; SDAG-NEXT: s_movk_i32 s4, 0x95
-; SDAG-NEXT: v_ashrrev_i32_e32 v7, 31, v5
-; SDAG-NEXT: v_or_b32_e32 v8, 1, v5
+; SDAG-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; SDAG-NEXT: v_or_b32_e32 v7, 1, v5
; SDAG-NEXT: v_or_b32_e32 v1, 0x800000, v0
-; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
+; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v8
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB2_3
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_add_u32_e32 v0, 0xffffff6a, v8
; SDAG-NEXT: v_mov_b32_e32 v2, 0
-; SDAG-NEXT: v_sub_u32_e32 v3, 0xd6, v9
-; SDAG-NEXT: v_add_u32_e32 v6, 0xffffff2a, v9
-; SDAG-NEXT: v_add_u32_e32 v0, 0xffffff6a, v9
+; SDAG-NEXT: v_sub_u32_e32 v3, 0xd6, v8
+; SDAG-NEXT: v_add_u32_e32 v8, 0xffffff2a, v8
; SDAG-NEXT: v_lshrrev_b64 v[3:4], v3, v[1:2]
-; SDAG-NEXT: v_lshlrev_b64 v[9:10], v6, v[1:2]
+; SDAG-NEXT: v_lshlrev_b64 v[8:9], v8, v[1:2]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
-; SDAG-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
; SDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; SDAG-NEXT: v_cndmask_b32_e64 v6, 0, v4, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v3, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v3, vcc
; SDAG-NEXT: v_lshlrev_b64 v[3:4], v0, v[1:2]
-; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, v9, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v3, vcc
-; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v8, 0
+; SDAG-NEXT: v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v3, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v7, 0
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
+; SDAG-NEXT: v_mul_lo_u32 v11, v6, v8
; SDAG-NEXT: v_mul_lo_u32 v12, v7, v9
-; SDAG-NEXT: v_mul_lo_u32 v6, v8, v6
-; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v8, v[1:2]
-; SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v8, v9, 0
+; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v7, v[1:2]
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v7, v8, 0
; SDAG-NEXT: v_mov_b32_e32 v1, v3
-; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v7, v[1:2]
-; SDAG-NEXT: v_add3_u32 v10, v10, v6, v12
-; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v11, v[9:10]
-; SDAG-NEXT: v_mul_lo_u32 v6, v5, v13
+; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v6, v[1:2]
+; SDAG-NEXT: v_add3_u32 v9, v9, v12, v11
+; SDAG-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v5, v10, v[8:9]
+; SDAG-NEXT: v_mul_lo_u32 v9, v5, v13
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; SDAG-NEXT: v_mul_lo_u32 v5, v5, v11
-; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v7, v[2:3]
-; SDAG-NEXT: ; implicit-def: $vgpr7
-; SDAG-NEXT: v_add3_u32 v4, v5, v9, v6
-; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v8
+; SDAG-NEXT: v_mul_lo_u32 v5, v5, v10
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v6, v[2:3]
+; SDAG-NEXT: ; implicit-def: $vgpr6
+; SDAG-NEXT: v_add3_u32 v4, v5, v8, v9
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v7
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: ; implicit-def: $vgpr8
-; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
+; SDAG-NEXT: ; implicit-def: $vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr5
; SDAG-NEXT: .LBB2_3: ; %Flow
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
; SDAG-NEXT: s_cbranch_execz .LBB2_5
; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; SDAG-NEXT: v_sub_u32_e32 v0, 0x96, v9
+; SDAG-NEXT: v_sub_u32_e32 v0, 0x96, v8
; SDAG-NEXT: v_lshrrev_b32_e32 v3, v0, v1
-; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v3, v8, 0
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v3, v7, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mov_b32_e32 v8, v2
-; SDAG-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v3, v7, v[1:2]
+; SDAG-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v3, v6, v[1:2]
; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[8:9], v5, v3, v[7:8]
; SDAG-NEXT: v_mov_b32_e32 v1, v6
; SDAG-NEXT: .LBB2_5: ; %Flow1
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index 82ae949d0c8f5..655b6e811933c 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -50,18 +50,18 @@ define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX10-NEXT: s_sub_i32 s3, 0, s6
+; GFX10-NEXT: s_sub_i32 s2, 0, s6
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_mul_i32 s3, s3, s2
-; GFX10-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX10-NEXT: s_mov_b32 s3, 0
-; GFX10-NEXT: s_add_i32 s4, s2, s4
+; GFX10-NEXT: s_mul_i32 s2, s2, s4
+; GFX10-NEXT: s_mul_hi_u32 s2, s4, s2
+; GFX10-NEXT: s_add_i32 s4, s4, s2
; GFX10-NEXT: s_mov_b32 s2, s3
; GFX10-NEXT: .LBB0_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -93,21 +93,22 @@ define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX11-NEXT: s_sub_i32 s3, 0, s6
+; GFX11-NEXT: s_sub_i32 s2, 0, s6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_mul_i32 s3, s3, s2
-; GFX11-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_add_i32 s4, s2, s4
+; GFX11-NEXT: s_mul_i32 s2, s2, s4
+; GFX11-NEXT: s_mul_hi_u32 s2, s4, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s4, s4, s2
; GFX11-NEXT: s_mov_b32 s2, s3
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB0_1: ; %bb3
@@ -198,18 +199,18 @@ define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX10-NEXT: s_sub_i32 s3, 0, s6
+; GFX10-NEXT: s_sub_i32 s2, 0, s6
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_mul_i32 s3, s3, s2
-; GFX10-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX10-NEXT: s_mov_b32 s3, 0
-; GFX10-NEXT: s_add_i32 s4, s2, s4
+; GFX10-NEXT: s_mul_i32 s2, s2, s4
+; GFX10-NEXT: s_mul_hi_u32 s2, s4, s2
+; GFX10-NEXT: s_add_i32 s4, s4, s2
; GFX10-NEXT: s_mov_b32 s2, s3
; GFX10-NEXT: .LBB1_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -239,21 +240,22 @@ define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX11-NEXT: s_sub_i32 s3, 0, s6
+; GFX11-NEXT: s_sub_i32 s2, 0, s6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_mul_i32 s3, s3, s2
-; GFX11-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_add_i32 s4, s2, s4
+; GFX11-NEXT: s_mul_i32 s2, s2, s4
+; GFX11-NEXT: s_mul_hi_u32 s2, s4, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s4, s4, s2
; GFX11-NEXT: s_mov_b32 s2, s3
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB1_1: ; %bb3
diff --git a/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll b/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
index 442bcb1990bbd..f2d5cdabb5517 100644
--- a/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
@@ -14,11 +14,11 @@ define amdgpu_kernel void @stepper_test_kernel_DType_I6A6AcB6A6AsA6A6A_68a5362b9
; GFX942-NEXT: s_cbranch_scc1 .LBB0_3
; GFX942-NEXT: ; %bb.1: ; %.lr.ph.preheader
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_mov_b32 s8, 0x47004600
; GFX942-NEXT: s_mov_b32 s9, 0x45004400
; GFX942-NEXT: s_mov_b32 s10, 0x42004000
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: .LBB0_2: ; %.lr.ph
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
index 0c7d314c3531d..4d9a7fc63331c 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
@@ -409,9 +409,9 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr4
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: ; implicit-def: $vgpr2
; SDAG-NEXT: s_branch .LBB1_2
; SDAG-NEXT: .LBB1_14: ; %itofp-if-then20
; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index 5efb825a7a90a..54f16e2a586ca 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -388,9 +388,9 @@ define float @uitofp_i128_to_f32(i128 %x) {
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr4
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: ; implicit-def: $vgpr2
; SDAG-NEXT: s_branch .LBB1_2
; SDAG-NEXT: .LBB1_14: ; %itofp-if-then20
; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3
@@ -592,27 +592,26 @@ define double @sitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v10, v0, v10
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v13, v[6:7]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v12
-; SDAG-NEXT: v_add_u32_e32 v16, 55, v9
; SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc
; SDAG-NEXT: v_lshrrev_b64 v[10:11], v12, v[6:7]
-; SDAG-NEXT: v_lshrrev_b64 v[12:13], v13, v[4:5]
-; SDAG-NEXT: v_lshlrev_b64 v[14:15], v16, v[6:7]
+; SDAG-NEXT: v_add_u32_e32 v15, 55, v9
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v12
+; SDAG-NEXT: v_lshrrev_b64 v[11:12], v13, v[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[13:14], v15, v[6:7]
; SDAG-NEXT: v_add_u32_e32 v9, -9, v9
-; SDAG-NEXT: v_or_b32_e32 v15, v15, v13
; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
-; SDAG-NEXT: v_lshlrev_b64 v[12:13], v9, v[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
+; SDAG-NEXT: v_or_b32_e32 v13, v13, v11
+; SDAG-NEXT: v_lshlrev_b64 v[11:12], v9, v[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v4, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; SDAG-NEXT: v_lshlrev_b64 v[4:5], v16, v[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v9, v12, v14, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
+; SDAG-NEXT: v_lshlrev_b64 v[4:5], v15, v[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v11, v13, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v6, v9, v6, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
@@ -630,10 +629,10 @@ define double @sitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: .LBB2_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %itofp-sw-bb
-; SDAG-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; SDAG-NEXT: v_lshrrev_b32_e32 v0, 31, v5
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], 1, v[6:7]
+; SDAG-NEXT: v_lshrrev_b32_e32 v1, 31, v5
; SDAG-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
-; SDAG-NEXT: v_or_b32_e32 v6, v6, v0
+; SDAG-NEXT: v_or_b32_e32 v6, v0, v1
; SDAG-NEXT: ; %bb.9: ; %itofp-sw-epilog
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: v_lshrrev_b32_e32 v0, 2, v4
@@ -668,7 +667,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: ; implicit-def: $vgpr2
; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
-; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: ; implicit-def: $vgpr5
; SDAG-NEXT: s_branch .LBB2_2
; SDAG-NEXT: .LBB2_14: ; %itofp-if-then20
; SDAG-NEXT: v_lshrrev_b64 v[0:1], 3, v[4:5]
@@ -889,27 +888,26 @@ define double @uitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v9, v4, v9
; SDAG-NEXT: v_lshrrev_b64 v[4:5], v12, v[2:3]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v11
-; SDAG-NEXT: v_add_u32_e32 v15, 55, v8
; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
; SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc
; SDAG-NEXT: v_lshrrev_b64 v[9:10], v11, v[2:3]
-; SDAG-NEXT: v_lshrrev_b64 v[11:12], v12, v[0:1]
-; SDAG-NEXT: v_lshlrev_b64 v[13:14], v15, v[2:3]
+; SDAG-NEXT: v_add_u32_e32 v14, 55, v8
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; SDAG-NEXT: v_lshrrev_b64 v[10:11], v12, v[0:1]
+; SDAG-NEXT: v_lshlrev_b64 v[12:13], v14, v[2:3]
; SDAG-NEXT: v_add_u32_e32 v8, -9, v8
-; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
; SDAG-NEXT: v_or_b32_e32 v13, v13, v11
-; SDAG-NEXT: v_lshlrev_b64 v[11:12], v8, v[0:1]
-; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
+; SDAG-NEXT: v_or_b32_e32 v12, v12, v10
+; SDAG-NEXT: v_lshlrev_b64 v[10:11], v8, v[0:1]
; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v15
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
; SDAG-NEXT: v_cndmask_b32_e64 v5, v5, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v4, v4, v0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v8, v12, v14, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v15
-; SDAG-NEXT: v_lshlrev_b64 v[0:1], v15, v[0:1]
-; SDAG-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; SDAG-NEXT: v_lshlrev_b64 v[0:1], v14, v[0:1]
+; SDAG-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v10, v12, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
@@ -962,9 +960,9 @@ define double @uitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr6
-; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: ; implicit-def: $vgpr6
; SDAG-NEXT: s_branch .LBB3_2
; SDAG-NEXT: .LBB3_14: ; %itofp-if-then20
; SDAG-NEXT: v_lshrrev_b64 v[4:5], 3, v[0:1]
@@ -1515,9 +1513,9 @@ define half @uitofp_i128_to_f16(i128 %x) {
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
-; SDAG-NEXT: ; implicit-def: $vgpr4
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: ; implicit-def: $vgpr4
+; SDAG-NEXT: ; implicit-def: $vgpr2
; SDAG-NEXT: s_branch .LBB5_2
; SDAG-NEXT: .LBB5_14: ; %itofp-if-then20
; SDAG-NEXT: v_alignbit_b32 v7, v1, v0, 3
diff --git a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
index 8693dd1236e5f..ad319b4fb8cae 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
@@ -9,10 +9,8 @@ define <3 x float> @liveout_undef_subrange(<3 x float> %arg) {
; CHECK-LABEL: liveout_undef_subrange:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_add_f32_e32 v3, v2, v2
+; CHECK-NEXT: v_add_f32_e32 v0, v2, v2
; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: ; kill: killed $vgpr1
-; CHECK-NEXT: v_add_f32_e32 v0, v0, v0
; CHECK-NEXT: .LBB0_1: ; %bb1
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_cmp_neq_f32_e32 vcc, 0, v2
@@ -22,7 +20,7 @@ define <3 x float> @liveout_undef_subrange(<3 x float> %arg) {
; CHECK-NEXT: ; %bb.2: ; %bb2
; CHECK-NEXT: ; in Loop: Header=BB0_1 Depth=1
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_mul_f32_e32 v2, v3, v2
+; CHECK-NEXT: v_mul_f32_e32 v2, v0, v2
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_cbranch_execnz .LBB0_1
; CHECK-NEXT: ; %bb.3: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index 61f79ccd55f1c..90b6f653ec066 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -12768,11 +12768,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_mov_b32_e32 v88, v1
-; ALIGNED-NEXT: v_mov_b32_e32 v87, v0
+; ALIGNED-NEXT: v_mov_b32_e32 v84, v1
+; ALIGNED-NEXT: v_mov_b32_e32 v83, v0
; ALIGNED-NEXT: s_mov_b32 s4, exec_lo
-; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[87:88]
-; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v87, vcc_lo
+; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[83:84]
+; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v83, vcc_lo
; ALIGNED-NEXT: v_cmpx_ge_u32_e32 v2, v3
; ALIGNED-NEXT: s_xor_b32 s6, exec_lo, s4
; ALIGNED-NEXT: s_cbranch_execz .LBB9_3
@@ -12836,7 +12836,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:71
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:72
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:73
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:74
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:76
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:77
; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:78
@@ -12963,7 +12963,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:75
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v70
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
@@ -12992,7 +12992,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:91
@@ -13014,10 +13014,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v4, v0, 8, v80
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v82, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v83, 8, v84
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v85, 8, v86
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v1
@@ -13242,92 +13242,92 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:141
-; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:160
-; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:161
-; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:162
-; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:163
-; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:160
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:164
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:165
; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:167
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:168
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:169
-; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:170
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:172
-; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:173
-; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:174
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:175
-; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:176
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:177
-; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:178
-; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:180
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:181
-; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:182
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:183
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:184
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:185
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:186
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:188
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:190
-; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:191
-; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:192
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:193
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:194
-; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:195
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:196
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:197
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:198
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:199
-; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:200
-; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:201
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:202
-; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:204
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:205
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:206
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:207
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:208
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:209
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:210
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:212
-; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:229
-; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:230
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:231
-; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:232
-; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:233
-; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:234
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:235
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:236
-; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:237
; ALIGNED-NEXT: s_clause 0x17
-; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:238
-; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:239
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:240
-; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:241
-; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:242
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:243
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:244
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:245
-; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:246
-; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:247
-; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:248
-; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:249
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:250
-; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:253
-; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:254
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:255
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen
+; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen
; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:3
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:4
+; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:4
; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:5
-; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:6
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:6
; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:7
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
@@ -13363,16 +13363,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:151
; ALIGNED-NEXT: s_waitcnt vmcnt(21)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v46, 8, v56
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v42, 8, v44
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v101, 8, v112
-; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: buffer_store_dword v79, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v96, 8, v99
+; ALIGNED-NEXT: s_waitcnt vmcnt(10)
+; ALIGNED-NEXT: buffer_store_dword v107, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:149
-; ALIGNED-NEXT: buffer_store_dword v92, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(10)
; ALIGNED-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v77, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(9)
+; ALIGNED-NEXT: buffer_store_dword v75, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: buffer_store_dword v79, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
@@ -13393,20 +13396,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:213
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:214
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:215
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:216
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:217
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:218
-; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:219
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:220
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:159
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:157
-; ALIGNED-NEXT: s_waitcnt vmcnt(10)
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
@@ -13417,232 +13411,245 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT: s_clause 0x7
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:220
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v123
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v97, 8, v95
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v95, 8, v122
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v90
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v57, 8, v107
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v57, 8, v105
; ALIGNED-NEXT: v_lshl_or_b32 v4, v47, 8, v59
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v100, 8, v113
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 8, v102
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v102, 8, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v87, 8, v100
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v43, 8, v45
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v117, 8, v115
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v41, 8, v114
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v85
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v81
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v81, 8, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v80, 8, v71
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v69, 8, v84
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v68, 8, v70
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v82
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v68
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v55
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v54
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v51, 8, v53
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v50, 8, v52
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v66, 8, v67
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v64, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v64, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v54, 8, v55
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v48, 8, v51
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v38, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v38, 8, v49
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v37
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:221
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:222
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:223
-; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:224
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:225
-; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:226
; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:227
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:228
-; ALIGNED-NEXT: v_lshl_or_b32 v74, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v35, 8, v49
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v36
-; ALIGNED-NEXT: v_lshl_or_b32 v41, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v25, 8, v28
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v24, 8, v26
-; ALIGNED-NEXT: v_lshl_or_b32 v80, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v33
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v29
-; ALIGNED-NEXT: v_lshl_or_b32 v71, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v18
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v11, 8, v14
-; ALIGNED-NEXT: v_lshl_or_b32 v50, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v15, 8, v17
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v12
-; ALIGNED-NEXT: v_lshl_or_b32 v37, v4, 16, v3
+; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT: v_lshl_or_b32 v72, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v33, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v32, 8, v34
+; ALIGNED-NEXT: v_lshl_or_b32 v116, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v23, 8, v26
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v24
+; ALIGNED-NEXT: v_lshl_or_b32 v70, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v29, 8, v31
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v28, 8, v27
+; ALIGNED-NEXT: v_lshl_or_b32 v69, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(15)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v11, 8, v16
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v48, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(11)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v15
+; ALIGNED-NEXT: s_waitcnt vmcnt(9)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v35, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v16
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v14
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v8
-; ALIGNED-NEXT: v_lshl_or_b32 v32, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v21, 8, v23
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v20, 8, v22
-; ALIGNED-NEXT: v_lshl_or_b32 v27, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v30, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v18, 8, v20
+; ALIGNED-NEXT: v_lshl_or_b32 v25, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v126, 8, v0
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v126
-; ALIGNED-NEXT: v_lshl_or_b32 v19, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v123
+; ALIGNED-NEXT: v_lshl_or_b32 v17, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v1
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v110, 8, v120
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v78, 8, v94
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v90, 8, v89
-; ALIGNED-NEXT: v_lshl_or_b32 v62, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v106, 8, v108
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v104
-; ALIGNED-NEXT: v_lshl_or_b32 v60, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v75, 8, v91
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v72, 8, v76
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v110, 8, v121
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v106, 8, v108
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v77, 8, v76
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v89, 8, v88
+; ALIGNED-NEXT: v_lshl_or_b32 v58, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v94, 8, v104
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v92, 8, v91
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v73, 8, v78
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v62, 8, v74
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v58, 8, v73
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v103, 16, v4
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v114, 8, v116
-; ALIGNED-NEXT: v_lshl_or_b32 v98, v119, 16, v103
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v42, 8, v44
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v118, 8, v40
-; ALIGNED-NEXT: v_lshl_or_b32 v99, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v46, 8, v60
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v101, 16, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v113, 8, v112
+; ALIGNED-NEXT: v_lshl_or_b32 v97, v118, 16, v101
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v119, 8, v40
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v115, 8, v117
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v118, 16, v101
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:1
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v101, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v119, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v103, 8, v96
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v61, 8, v119
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v79, 8, v77
+; ALIGNED-NEXT: buffer_store_dword v118, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v101, 8, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v61, 8, v118
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v118, 16, v101
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v63, 8, v107
; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:12
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:13
-; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:14
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:15
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:11
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v63, 8, v92
+; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v79, 8, v75
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:8
-; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:9
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:8
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:9
; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:10
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v118, 16, v101
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v109, 8, v125
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v107, 8, v125
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v111, 8, v121
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v109, 8, v111
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v118, 16, v101
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v77, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v101, v75, 8, v79
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v79, 8, v63
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v119, 16, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v120, 8, v63
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v118, 16, v101
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:18
; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:17
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v97, off, s[0:3], s32 offset:236
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224
-; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v87, 3
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v88, vcc_lo
-; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:247
-; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:248
-; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:246
-; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:252
-; ALIGNED-NEXT: flat_store_byte v[3:4], v114 offset:250
-; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:251
-; ALIGNED-NEXT: flat_store_byte v[3:4], v116 offset:249
-; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:245
-; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:239
-; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:238
-; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:244
-; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:242
-; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:243
-; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:241
-; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:237
-; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:231
-; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:232
-; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:230
-; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:236
-; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:234
-; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:235
-; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:233
-; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:229
-; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:223
+; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v83, 3
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v84, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:247
+; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:248
+; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:246
+; ALIGNED-NEXT: flat_store_byte v[3:4], v96 offset:252
+; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:250
+; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:251
+; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:249
+; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:245
+; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:239
+; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:238
+; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:244
+; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:242
+; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:243
+; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:241
+; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:237
+; ALIGNED-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:231
+; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:232
+; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:230
+; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:236
+; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:234
+; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:235
+; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:233
+; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:229
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:223
; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:224
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:222
-; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:228
-; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:226
-; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:227
-; ALIGNED-NEXT: flat_store_byte v[3:4], v1 offset:225
-; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:221
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:192
-; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:206
-; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:208
-; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:207
-; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:210
-; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:212
-; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:211
-; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:209
-; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:215
+; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:222
+; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:228
+; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:226
+; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:227
+; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:225
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:221
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:206
+; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:207
+; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:210
+; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:211
+; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:209
+; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:215
; ALIGNED-NEXT: flat_store_byte v[3:4], v10 offset:216
-; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:214
-; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:220
+; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:220
; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:218
-; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:219
-; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:217
-; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:213
-; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:205
-; ALIGNED-NEXT: buffer_store_dword v71, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v74, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:219
+; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:217
+; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:205
+; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v70, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v72, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:199
-; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:200
-; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:198
-; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:204
-; ALIGNED-NEXT: flat_store_byte v[3:4], v25 offset:202
-; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:203
-; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:201
-; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:197
-; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:191
-; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:192
-; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:190
-; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v35 offset:194
-; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:195
-; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:193
-; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:189
+; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:199
+; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:200
+; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:198
+; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:204
+; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:202
+; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:203
+; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:201
+; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:197
+; ALIGNED-NEXT: flat_store_byte v[3:4], v37 offset:191
+; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:192
+; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:190
+; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:196
+; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:194
+; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:195
+; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:193
+; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:189
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v127, 8, v119
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v127, 8, v118
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v127, v103, 8, v61
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v96, 16, v127
+; ALIGNED-NEXT: v_lshl_or_b32 v127, v101, 8, v61
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v86, 16, v127
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
@@ -13655,22 +13662,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:183
-; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:184
-; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:182
-; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:188
-; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:186
-; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:187
-; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:185
-; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:181
-; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:175
-; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:176
-; ALIGNED-NEXT: flat_store_byte v[3:4], v83 offset:174
-; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:180
-; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:178
-; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:179
-; ALIGNED-NEXT: flat_store_byte v[3:4], v84 offset:177
-; ALIGNED-NEXT: flat_store_byte v[3:4], v85 offset:173
+; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:183
+; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:184
+; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:182
+; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:188
+; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:186
+; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:187
+; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:185
+; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:181
+; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:175
+; ALIGNED-NEXT: flat_store_byte v[3:4], v80 offset:176
+; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:180
+; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:178
+; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:179
+; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:177
+; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
@@ -13683,22 +13690,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:167
-; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:168
+; ALIGNED-NEXT: flat_store_byte v[3:4], v114 offset:167
+; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:168
; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:166
-; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:172
-; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:170
-; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:171
-; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:169
+; ALIGNED-NEXT: flat_store_byte v[3:4], v87 offset:172
+; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:170
+; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:171
+; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:169
; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:165
-; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:159
-; ALIGNED-NEXT: flat_store_byte v[3:4], v97 offset:160
-; ALIGNED-NEXT: flat_store_byte v[3:4], v105 offset:158
+; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:159
+; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:160
+; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:158
; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:164
; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:162
; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:163
-; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:161
-; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:157
+; ALIGNED-NEXT: flat_store_byte v[3:4], v105 offset:161
+; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
@@ -13817,7 +13824,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:128
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:128
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360
@@ -14057,7 +14064,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:64
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424
@@ -14165,7 +14172,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:32
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:36
@@ -14187,7 +14194,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:384
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:23
@@ -14212,11 +14219,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:21
-; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:15
+; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:15
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:16
-; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:14
+; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:14
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:20
@@ -14229,7 +14236,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[87:88], v61 offset:16
+; ALIGNED-NEXT: flat_store_byte v[83:84], v61 offset:16
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408
@@ -14244,27 +14251,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:7
-; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:8
-; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:10
-; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:6
-; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:12
-; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:11
+; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:8
+; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:10
+; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:6
+; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:12
+; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:11
; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:9
-; ALIGNED-NEXT: flat_store_byte v[87:88], v92 offset:8
+; ALIGNED-NEXT: flat_store_byte v[83:84], v79 offset:8
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:2
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:2
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:1
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:1
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0 offset:4
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:4
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[87:88], v0
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_co_u32 v87, vcc_lo, 0x100, v87
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v88, null, 0, v88, vcc_lo
+; ALIGNED-NEXT: v_add_co_u32 v83, vcc_lo, 0x100, v83
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v84, null, 0, v84, vcc_lo
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
@@ -14281,8 +14288,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_andn2_saveexec_b32 s6, s6
; ALIGNED-NEXT: s_cbranch_execz .LBB9_6
; ALIGNED-NEXT: ; %bb.4: ; %memmove_bwd_loop.preheader
-; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0x700, v87
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, 0, v88, vcc_lo
+; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0x700, v83
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, 0, v84, vcc_lo
; ALIGNED-NEXT: s_movk_i32 s4, 0xf800
; ALIGNED-NEXT: s_mov_b32 s5, -1
; ALIGNED-NEXT: .LBB9_5: ; %memmove_bwd_loop
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index eb8eb8b4f1cb2..4a2fc188d7c35 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -10,13 +10,19 @@ define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align
; CHECK-LABEL: memmove_p0_p0:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB0_3
@@ -31,27 +37,21 @@ define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB0_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB0_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_dwordx4 v[14:17], v[4:5]
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: flat_load_dwordx4 v[15:18], v[11:12]
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: flat_store_dwordx4 v[13:14], v[15:18]
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB0_5
; CHECK-NEXT: .LBB0_6: ; %Flow30
@@ -59,20 +59,20 @@ define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB0_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB0_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: flat_load_ubyte v4, v[2:3]
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v4
@@ -82,36 +82,37 @@ define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_cbranch_execnz .LBB0_8
; CHECK-NEXT: .LBB0_9: ; %Flow28
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB0_2
; CHECK-NEXT: .LBB0_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB0_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v3, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB0_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_ubyte v12, v[10:11]
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: flat_load_ubyte v6, v[11:12]
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_byte v[4:5], v12
+; CHECK-NEXT: flat_store_byte v[4:5], v6
; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -129,19 +130,19 @@ define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB0_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; CHECK-NEXT: flat_store_dwordx4 v[11:12], v[4:7]
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB0_15
; CHECK-NEXT: .LBB0_16: ; %Flow32
@@ -158,13 +159,19 @@ define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p0_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB1_3
@@ -179,27 +186,21 @@ define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB1_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB1_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: global_load_dwordx4 v[15:18], v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: flat_store_dwordx4 v[13:14], v[15:18]
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB1_5
; CHECK-NEXT: .LBB1_6: ; %Flow32
@@ -207,20 +208,20 @@ define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB1_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB1_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: global_load_ubyte v4, v[2:3], off
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v4
@@ -230,38 +231,39 @@ define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_cbranch_execnz .LBB1_8
; CHECK-NEXT: .LBB1_9: ; %Flow30
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB1_2
; CHECK-NEXT: .LBB1_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB1_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB1_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_ubyte v12, v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
+; CHECK-NEXT: global_load_ubyte v6, v[4:5], off
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_byte v[10:11], v12
-; CHECK-NEXT: v_add_co_u32 v10, s5, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s5
+; CHECK-NEXT: flat_store_byte v[11:12], v6
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB1_12
; CHECK-NEXT: .LBB1_13: ; %Flow36
@@ -277,19 +279,19 @@ define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB1_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; CHECK-NEXT: flat_store_dwordx4 v[11:12], v[4:7]
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB1_15
; CHECK-NEXT: .LBB1_16: ; %Flow34
@@ -306,16 +308,20 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p0_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; CHECK-NEXT: v_and_b32_e32 v5, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v11, v1
+; CHECK-NEXT: v_mov_b32_e32 v10, v0
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cndmask_b32_e32 v12, -1, v0, vcc_lo
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
-; CHECK-NEXT: v_cndmask_b32_e32 v9, -1, v0, vcc_lo
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[5:6]
-; CHECK-NEXT: v_cmpx_ge_u32_e32 v2, v9
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: v_mov_b32_e32 v8, v3
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
+; CHECK-NEXT: v_cmpx_ge_u32_e32 v2, v12
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB2_3
; CHECK-NEXT: ; %bb.1: ; %Flow39
@@ -329,25 +335,21 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB2_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v10, v1
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
-; CHECK-NEXT: v_mov_b32_e32 v12, v6
-; CHECK-NEXT: v_mov_b32_e32 v11, v5
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
+; CHECK-NEXT: v_mov_b32_e32 v5, v2
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB2_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[13:16], v4
-; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]
+; CHECK-NEXT: ds_read_b128 v[12:15], v5
+; CHECK-NEXT: v_add_co_u32 v8, s5, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 16, v5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[9:10], v[13:16]
-; CHECK-NEXT: v_add_co_u32 v9, s6, v9, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s6
+; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[12:15]
+; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB2_5
; CHECK-NEXT: .LBB2_6: ; %Flow34
@@ -355,19 +357,18 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB2_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v5
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v6, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB2_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v3, v2
-; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
+; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v3
@@ -377,36 +378,37 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_cbranch_execnz .LBB2_8
; CHECK-NEXT: .LBB2_9: ; %Flow32
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
-; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CHECK-NEXT: ; implicit-def: $vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr2
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB2_2
; CHECK-NEXT: .LBB2_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB2_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v9, s4, v0, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v1, v4, s4
-; CHECK-NEXT: v_add3_u32 v4, v3, v2, -1
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v5
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v4, s4
+; CHECK-NEXT: v_add3_u32 v10, v5, v2, -1
+; CHECK-NEXT: v_add_co_u32 v4, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v12, s4
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB2_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_u8 v11, v4
-; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: ds_read_u8 v11, v10
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
+; CHECK-NEXT: v_add_nc_u32_e32 v10, -1, v10
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: flat_store_byte v[9:10], v11
-; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: flat_store_byte v[4:5], v11
+; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB2_12
; CHECK-NEXT: .LBB2_13: ; %Flow38
@@ -414,26 +416,25 @@ define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB2_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; CHECK-NEXT: v_add3_u32 v2, v3, v2, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB2_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[7:10], v2
-; CHECK-NEXT: v_mov_b32_e32 v3, v5
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
+; CHECK-NEXT: ds_read_b128 v[3:6], v2
+; CHECK-NEXT: v_mov_b32_e32 v10, v8
+; CHECK-NEXT: v_mov_b32_e32 v11, v9
; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
-; CHECK-NEXT: v_add_co_u32 v5, vcc_lo, v3, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v4, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v3, s4, v0, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v1, v4, s4
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[5:6]
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[3:4], v[7:10]
+; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[3:6]
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB2_15
; CHECK-NEXT: .LBB2_16: ; %Flow36
@@ -450,13 +451,19 @@ define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p0_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB3_3
@@ -471,27 +478,21 @@ define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB3_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB3_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: global_load_dwordx4 v[15:18], v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: flat_store_dwordx4 v[13:14], v[15:18]
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB3_5
; CHECK-NEXT: .LBB3_6: ; %Flow31
@@ -499,20 +500,20 @@ define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB3_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB3_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: global_load_ubyte v4, v[2:3], off
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v4
@@ -522,36 +523,37 @@ define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_cbranch_execnz .LBB3_8
; CHECK-NEXT: .LBB3_9: ; %Flow29
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB3_2
; CHECK-NEXT: .LBB3_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB3_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v3, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB3_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_ubyte v12, v[10:11], off
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: global_load_ubyte v6, v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_byte v[4:5], v12
+; CHECK-NEXT: flat_store_byte v[4:5], v6
; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -569,19 +571,19 @@ define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB3_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; CHECK-NEXT: flat_store_dwordx4 v[11:12], v[4:7]
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB3_15
; CHECK-NEXT: .LBB3_16: ; %Flow33
@@ -598,16 +600,20 @@ define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p0_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; CHECK-NEXT: v_and_b32_e32 v5, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v11, v1
+; CHECK-NEXT: v_mov_b32_e32 v10, v0
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cndmask_b32_e32 v12, -1, v0, vcc_lo
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
-; CHECK-NEXT: v_cndmask_b32_e32 v9, -1, v0, vcc_lo
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[5:6]
-; CHECK-NEXT: v_cmpx_ge_u32_e32 v2, v9
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: v_mov_b32_e32 v8, v3
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
+; CHECK-NEXT: v_cmpx_ge_u32_e32 v2, v12
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB4_3
; CHECK-NEXT: ; %bb.1: ; %Flow39
@@ -621,29 +627,25 @@ define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB4_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v10, v1
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
-; CHECK-NEXT: v_mov_b32_e32 v12, v6
-; CHECK-NEXT: v_mov_b32_e32 v11, v5
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
+; CHECK-NEXT: v_mov_b32_e32 v5, v2
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB4_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v13, v4, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v14, v4, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v15, v4, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v16, v4, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]
+; CHECK-NEXT: buffer_load_dword v12, v5, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v13, v5, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v14, v5, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v15, v5, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v8, s5, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 16, v5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[9:10], v[13:16]
-; CHECK-NEXT: v_add_co_u32 v9, s6, v9, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s6
+; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[12:15]
+; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB4_5
; CHECK-NEXT: .LBB4_6: ; %Flow34
@@ -651,19 +653,18 @@ define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB4_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v5
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v6, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB4_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
+; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_byte v[0:1], v3
@@ -673,36 +674,37 @@ define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_cbranch_execnz .LBB4_8
; CHECK-NEXT: .LBB4_9: ; %Flow32
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
-; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CHECK-NEXT: ; implicit-def: $vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr2
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB4_2
; CHECK-NEXT: .LBB4_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB4_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v9, s4, v0, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v1, v4, s4
-; CHECK-NEXT: v_add3_u32 v4, v3, v2, -1
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v5
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v4, s4
+; CHECK-NEXT: v_add3_u32 v10, v5, v2, -1
+; CHECK-NEXT: v_add_co_u32 v4, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v12, s4
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB4_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: buffer_load_ubyte v11, v4, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: buffer_load_ubyte v11, v10, s[0:3], 0 offen
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
+; CHECK-NEXT: v_add_nc_u32_e32 v10, -1, v10
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_byte v[9:10], v11
-; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: flat_store_byte v[4:5], v11
+; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB4_12
; CHECK-NEXT: .LBB4_13: ; %Flow38
@@ -710,30 +712,29 @@ define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB4_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; CHECK-NEXT: v_add3_u32 v2, v3, v2, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB4_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_mov_b32_e32 v3, v5
-; CHECK-NEXT: v_mov_b32_e32 v4, v6
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_mov_b32_e32 v10, v8
+; CHECK-NEXT: v_mov_b32_e32 v11, v9
; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
-; CHECK-NEXT: v_add_co_u32 v5, vcc_lo, v3, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v4, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v3, s4, v0, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v1, v4, s4
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[5:6]
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[3:4], v[7:10]
+; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[3:6]
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB4_15
; CHECK-NEXT: .LBB4_16: ; %Flow36
@@ -751,13 +752,19 @@ define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align
; CHECK-LABEL: memmove_p1_p0:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB5_3
@@ -771,27 +778,21 @@ define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB5_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB5_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_dwordx4 v[14:17], v[4:5]
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: flat_load_dwordx4 v[15:18], v[11:12]
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[15:18], off
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB5_5
; CHECK-NEXT: .LBB5_6: ; %Flow32
@@ -799,20 +800,20 @@ define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB5_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB5_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: flat_load_ubyte v4, v[2:3]
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: global_store_byte v[0:1], v4, off
@@ -822,36 +823,37 @@ define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_cbranch_execnz .LBB5_8
; CHECK-NEXT: .LBB5_9: ; %Flow30
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB5_2
; CHECK-NEXT: .LBB5_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB5_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v3, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB5_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_ubyte v12, v[10:11]
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: flat_load_ubyte v6, v[11:12]
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: global_store_byte v[4:5], v12, off
+; CHECK-NEXT: global_store_byte v[4:5], v6, off
; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -869,19 +871,19 @@ define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB5_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; CHECK-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB5_15
; CHECK-NEXT: .LBB5_16: ; %Flow34
@@ -897,13 +899,19 @@ define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p1_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB6_3
@@ -917,27 +925,21 @@ define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB6_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB6_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: global_load_dwordx4 v[15:18], v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[15:18], off
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB6_5
; CHECK-NEXT: .LBB6_6: ; %Flow34
@@ -945,20 +947,20 @@ define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB6_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB6_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: global_load_ubyte v4, v[2:3], off
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_byte v[0:1], v4, off
@@ -968,36 +970,37 @@ define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_cbranch_execnz .LBB6_8
; CHECK-NEXT: .LBB6_9: ; %Flow32
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB6_2
; CHECK-NEXT: .LBB6_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB6_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v3, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB6_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_ubyte v12, v[10:11], off
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: global_load_ubyte v6, v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[4:5], v12, off
+; CHECK-NEXT: global_store_byte v[4:5], v6, off
; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -1015,19 +1018,19 @@ define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB6_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; CHECK-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB6_15
; CHECK-NEXT: .LBB6_16: ; %Flow36
@@ -1043,29 +1046,29 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p1_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB7_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v7, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[10:13], v9
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: ds_read_b128 v[8:11], v7
+; CHECK-NEXT: v_add_co_u32 v12, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_2
; CHECK-NEXT: .LBB7_3: ; %Flow9
@@ -1075,11 +1078,10 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB7_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v7, v2
@@ -1106,13 +1108,19 @@ define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p1_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v6, 15, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
-; CHECK-NEXT: v_and_b32_e32 v8, -16, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v5
+; CHECK-NEXT: v_mov_b32_e32 v6, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, v3
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v14, v1
+; CHECK-NEXT: v_and_b32_e32 v7, 15, v6
+; CHECK-NEXT: v_and_b32_e32 v4, -16, v6
+; CHECK-NEXT: v_mov_b32_e32 v13, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v5
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; CHECK-NEXT: v_cmpx_ge_u64_e32 v[2:3], v[0:1]
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB8_3
@@ -1126,27 +1134,21 @@ define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB8_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v11, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, v0
-; CHECK-NEXT: v_mov_b32_e32 v13, v9
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB8_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]
+; CHECK-NEXT: global_load_dwordx4 v[15:18], v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5
+; CHECK-NEXT: v_add_co_u32 v11, s5, v11, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, 0, v12, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[9:10]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off
-; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[15:18], off
+; CHECK-NEXT: v_add_co_u32 v13, s6, v13, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, 0, v14, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB8_5
; CHECK-NEXT: .LBB8_6: ; %Flow33
@@ -1154,20 +1156,20 @@ define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB8_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5
-; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8
-; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5
+; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v5, s5
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB8_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: global_load_ubyte v4, v[2:3], off
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
+; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5
; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s9, s5, s9
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_byte v[0:1], v4, off
@@ -1177,36 +1179,37 @@ define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_cbranch_execnz .LBB8_8
; CHECK-NEXT: .LBB8_9: ; %Flow31
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr9_vgpr10
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB8_2
; CHECK-NEXT: .LBB8_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB8_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v5, s4
; CHECK-NEXT: s_mov_b32 s8, 0
-; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v12, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v2, v6
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v3, v12, s4
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB8_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: global_load_ubyte v12, v[10:11], off
-; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
-; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: global_load_ubyte v6, v[11:12], off
+; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]
; CHECK-NEXT: s_or_b32 s8, s4, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[4:5], v12, off
+; CHECK-NEXT: global_store_byte v[4:5], v6, off
; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -1224,19 +1227,19 @@ define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB8_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v10, v8
; CHECK-NEXT: v_mov_b32_e32 v11, v9
-; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo
+; CHECK-NEXT: v_mov_b32_e32 v12, v10
+; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v11
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v12, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v11, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v12, vcc_lo
; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off
-; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v0, v11
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[9:10]
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v1, v12, s4
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; CHECK-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB8_15
; CHECK-NEXT: .LBB8_16: ; %Flow35
@@ -1252,34 +1255,34 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p1_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB9_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v7, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB9_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: buffer_load_dword v8, v7, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v12, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_2
; CHECK-NEXT: .LBB9_3: ; %Flow9
@@ -1289,11 +1292,10 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB9_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB9_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen
@@ -1321,18 +1323,22 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
; CHECK-LABEL: memmove_p3_p0:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_shared_base
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, s5, vcc_lo
-; CHECK-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc_lo
-; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[7:8]
-; CHECK-NEXT: v_cmpx_ge_u64_e32 v[1:2], v[9:10]
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, s5, vcc_lo
+; CHECK-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc_lo
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_mov_b32_e32 v10, v1
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ge_u64_e64 s5, v[1:2], v[8:9]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, v3
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
+; CHECK-NEXT: s_and_saveexec_b32 s6, s5
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB10_3
; CHECK-NEXT: ; %bb.1: ; %Flow39
@@ -1346,25 +1352,21 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB10_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v10, v2
-; CHECK-NEXT: v_mov_b32_e32 v9, v1
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
-; CHECK-NEXT: v_mov_b32_e32 v11, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v5, v0
; CHECK-NEXT: s_mov_b32 s9, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB10_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_dwordx4 v[13:16], v[9:10]
-; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5
-; CHECK-NEXT: v_add_co_u32 v9, s5, v9, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[11:12]
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[10:11]
+; CHECK-NEXT: v_add_co_u32 v8, s5, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, s5
+; CHECK-NEXT: v_add_co_u32 v10, s5, v10, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s9, s6, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: ds_write_b128 v4, v[13:16]
-; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4
+; CHECK-NEXT: ds_write_b128 v5, v[12:15]
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 16, v5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB10_5
; CHECK-NEXT: .LBB10_6: ; %Flow34
@@ -1372,58 +1374,58 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB10_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_add_nc_u32_e32 v5, v0, v3
+; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v3
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v4, s5
; CHECK-NEXT: s_mov_b32 s9, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v3
-; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v7
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v8, s5
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB10_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: flat_load_ubyte v2, v[0:1]
-; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
+; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
; CHECK-NEXT: v_add_co_u32 v0, s5, v0, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[5:6]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s9, s6, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: ds_write_b8 v3, v2
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; CHECK-NEXT: ds_write_b8 v5, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 1, v5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB10_8
; CHECK-NEXT: .LBB10_9: ; %Flow32
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
-; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CHECK-NEXT: ; implicit-def: $vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
-; CHECK-NEXT: ; implicit-def: $vgpr1_vgpr2
-; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr1
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB10_2
; CHECK-NEXT: .LBB10_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB10_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v9, s4, v1, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v2, v4, s4
-; CHECK-NEXT: v_add3_u32 v4, v3, v0, -1
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v1, v5
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v2, v4, s4
+; CHECK-NEXT: v_add3_u32 v10, v5, v0, -1
+; CHECK-NEXT: v_add_co_u32 v4, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v12, s4
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB10_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_ubyte v11, v[9:10]
-; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
+; CHECK-NEXT: flat_load_ubyte v11, v[4:5]
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: ds_write_b8 v4, v11
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
+; CHECK-NEXT: ds_write_b8 v10, v11
+; CHECK-NEXT: v_add_nc_u32_e32 v10, -1, v10
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB10_12
; CHECK-NEXT: .LBB10_13: ; %Flow38
@@ -1431,20 +1433,19 @@ define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB10_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo
; CHECK-NEXT: v_add3_u32 v0, v3, v0, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo
; CHECK-NEXT: s_mov_b32 s5, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB10_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v7
-; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v8, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v8
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v9, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, vcc_lo
; CHECK-NEXT: flat_load_dwordx4 v[3:6], v[3:4]
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[7:8]
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
; CHECK-NEXT: s_or_b32 s5, vcc_lo, s5
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: ds_write_b128 v0, v[3:6]
@@ -1465,29 +1466,29 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p3_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB11_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[8:9], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v7, v[8:11]
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB11_2
; CHECK-NEXT: .LBB11_3: ; %Flow9
@@ -1497,11 +1498,10 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB11_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
@@ -1529,13 +1529,13 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p3_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v4, v2
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v4
+; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; CHECK-NEXT: v_cmpx_ge_u32_e32 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_3
@@ -1550,16 +1550,18 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB12_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .LBB12_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[9:12], v3
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: ds_read_b128 v[9:12], v7
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b128 v8, v[9:12]
@@ -1571,17 +1573,16 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB12_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v2, -16, v2
-; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v2
; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .LBB12_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v2, v1
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b8 v0, v2
@@ -1590,31 +1591,32 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_cbranch_execnz .LBB12_8
; CHECK-NEXT: .LBB12_9: ; %Flow39
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: ; implicit-def: $vgpr2
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr4
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB12_2
; CHECK-NEXT: .LBB12_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s6, s4
; CHECK-NEXT: s_cbranch_execz .LBB12_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v4
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v4, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB12_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v8, v7
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b8 v6, v8
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: ds_write_b8 v4, v8
+; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB12_12
; CHECK-NEXT: .LBB12_13: ; %Flow45
@@ -1622,24 +1624,23 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB12_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
-; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v4, v0, v5
+; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v5, v1, v5
; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; CHECK-NEXT: .LBB12_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[5:8], v4
+; CHECK-NEXT: ds_read_b128 v[6:9], v5
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v5, -16, v5
; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; CHECK-NEXT: s_or_b32 s6, vcc_lo, s6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b128 v2, v[5:8]
-; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
+; CHECK-NEXT: ds_write_b128 v4, v[6:9]
+; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_15
; CHECK-NEXT: .LBB12_16: ; %Flow43
@@ -1656,29 +1657,29 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p3_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB13_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[8:9], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v7, v[8:11]
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB13_2
; CHECK-NEXT: .LBB13_3: ; %Flow9
@@ -1688,11 +1689,10 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB13_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
@@ -1721,53 +1721,52 @@ define void @memmove_p3_p5(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v4
+; CHECK-NEXT: v_and_b32_e32 v4, 15, v4
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[2:3]
; CHECK-NEXT: s_cbranch_execz .LBB14_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v7, v1
-; CHECK-NEXT: v_mov_b32_e32 v8, v0
+; CHECK-NEXT: v_mov_b32_e32 v6, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB14_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v12, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v8, v6, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v9, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v10, v6, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v6, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v6, 16, v6
; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v8, v[9:12]
-; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: ds_write_b128 v7, v[8:11]
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB14_2
; CHECK-NEXT: .LBB14_3: ; %Flow14
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB14_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v2
; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB14_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[4:5]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: ds_write_b8 v0, v2
@@ -1788,18 +1787,22 @@ define void @memmove_p5_p0(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align
; CHECK-LABEL: memmove_p5_p0:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, s5, vcc_lo
-; CHECK-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc_lo
-; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[7:8]
-; CHECK-NEXT: v_cmpx_ge_u64_e32 v[1:2], v[9:10]
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
+; CHECK-NEXT: v_mov_b32_e32 v11, v2
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cndmask_b32_e64 v9, 0, s5, vcc_lo
+; CHECK-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc_lo
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_mov_b32_e32 v10, v1
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ge_u64_e64 s5, v[1:2], v[8:9]
+; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_mov_b32_e32 v8, v3
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
+; CHECK-NEXT: s_and_saveexec_b32 s6, s5
; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB15_3
; CHECK-NEXT: ; %bb.1: ; %Flow39
@@ -1812,27 +1815,23 @@ define void @memmove_p5_p0(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s6, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB15_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v10, v2
-; CHECK-NEXT: v_mov_b32_e32 v9, v1
-; CHECK-NEXT: v_mov_b32_e32 v12, v8
-; CHECK-NEXT: v_mov_b32_e32 v11, v7
-; CHECK-NEXT: v_mov_b32_e32 v4, v0
+; CHECK-NEXT: v_mov_b32_e32 v5, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB15_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_dwordx4 v[13:16], v[9:10]
-; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5
-; CHECK-NEXT: v_add_co_u32 v9, s5, v9, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[10:11]
+; CHECK-NEXT: v_add_co_u32 v8, s5, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, s5
+; CHECK-NEXT: v_add_co_u32 v10, s5, v10, 16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s5
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[8:9]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: buffer_store_dword v16, v4, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v15, v4, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v14, v4, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v13, v4, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4
+; CHECK-NEXT: buffer_store_dword v15, v5, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v14, v5, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v13, v5, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v12, v5, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 16, v5
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB15_5
@@ -1841,58 +1840,58 @@ define void @memmove_p5_p0(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s8, s4
; CHECK-NEXT: s_cbranch_execz .LBB15_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_add_nc_u32_e32 v5, v0, v3
+; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v3
+; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v4, s5
; CHECK-NEXT: s_mov_b32 s9, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v3
-; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v7
-; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v8, s5
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB15_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: flat_load_ubyte v2, v[0:1]
-; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
+; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
; CHECK-NEXT: v_add_co_u32 v0, s5, v0, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s5
-; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[5:6]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s9, s6, s9
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: buffer_store_byte v2, v3, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 1, v3
+; CHECK-NEXT: buffer_store_byte v2, v5, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 1, v5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; CHECK-NEXT: s_cbranch_execnz .LBB15_8
; CHECK-NEXT: .LBB15_9: ; %Flow32
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8
-; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9
+; CHECK-NEXT: ; implicit-def: $vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
-; CHECK-NEXT: ; implicit-def: $vgpr1_vgpr2
-; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr1
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7
; CHECK-NEXT: s_cbranch_execz .LBB15_2
; CHECK-NEXT: .LBB15_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB15_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_co_u32 v9, s4, v1, v3
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v2, v4, s4
-; CHECK-NEXT: v_add3_u32 v4, v3, v0, -1
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
+; CHECK-NEXT: v_add_co_u32 v11, s4, v1, v5
+; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, v2, v4, s4
+; CHECK-NEXT: v_add3_u32 v10, v5, v0, -1
+; CHECK-NEXT: v_add_co_u32 v4, s4, v11, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v12, s4
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB15_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: flat_load_ubyte v11, v[9:10]
-; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
-; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
+; CHECK-NEXT: flat_load_ubyte v11, v[4:5]
+; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4
+; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: buffer_store_byte v11, v4, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
+; CHECK-NEXT: buffer_store_byte v11, v10, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v10, -1, v10
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execnz .LBB15_12
; CHECK-NEXT: .LBB15_13: ; %Flow38
@@ -1900,25 +1899,24 @@ define void @memmove_p5_p0(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align
; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB15_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo
; CHECK-NEXT: v_add3_u32 v0, v3, v0, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo
; CHECK-NEXT: s_mov_b32 s5, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB15_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v7
-; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v8, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v8
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v9, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v8, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v9, vcc_lo
; CHECK-NEXT: flat_load_dwordx4 v[3:6], v[3:4]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:12
; CHECK-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[7:8]
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]
; CHECK-NEXT: v_add_nc_u32_e32 v0, -16, v0
; CHECK-NEXT: s_or_b32 s5, vcc_lo, s5
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
@@ -1936,32 +1934,32 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p5_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB16_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB16_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[8:9], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v11, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v9, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v8, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB16_2
@@ -1972,11 +1970,10 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB16_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB16_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
@@ -2004,31 +2001,31 @@ define void @memmove_p5_p3(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v4, v2
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v4
+; CHECK-NEXT: v_and_b32_e32 v4, 15, v4
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[2:3]
; CHECK-NEXT: s_cbranch_execz .LBB17_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v7, v1
-; CHECK-NEXT: v_mov_b32_e32 v8, v0
+; CHECK-NEXT: v_mov_b32_e32 v6, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB17_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[9:12], v7
+; CHECK-NEXT: ds_read_b128 v[8:11], v6
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v6, 16, v6
; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: buffer_store_dword v11, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v9, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v8, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB17_2
@@ -2036,20 +2033,19 @@ define void @memmove_p5_p3(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB17_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v2
; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB17_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v2, v1
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[4:5]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen
@@ -2068,32 +2064,32 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p5_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB18_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v7, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB18_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v[8:9], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v11, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v9, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v8, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB18_2
@@ -2104,11 +2100,10 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
; CHECK-NEXT: s_cbranch_execz .LBB18_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
-; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB18_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s4
@@ -2135,13 +2130,13 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p5_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v4, v2
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v4
+; CHECK-NEXT: v_and_b32_e32 v2, -16, v4
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; CHECK-NEXT: v_cmpx_ge_u32_e32 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB19_3
@@ -2155,20 +2150,22 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB19_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v9, v3, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v10, v3, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v11, v3, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v3, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
+; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v12, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
@@ -2177,7 +2174,7 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -2187,17 +2184,16 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB19_9
; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v2, -16, v2
-; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v2
; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .LBB19_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen
@@ -2206,31 +2202,32 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_cbranch_execnz .LBB19_8
; CHECK-NEXT: .LBB19_9: ; %Flow39
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7
-; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: ; implicit-def: $vgpr2
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
+; CHECK-NEXT: ; implicit-def: $vgpr4
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB19_2
; CHECK-NEXT: .LBB19_10: ; %memmove_copy_backwards
; CHECK-NEXT: s_and_saveexec_b32 s6, s4
; CHECK-NEXT: s_cbranch_execz .LBB19_13
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
-; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v4
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v4, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB19_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v8, v7, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_byte v8, v6, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: buffer_store_byte v8, v4, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB19_12
; CHECK-NEXT: .LBB19_13: ; %Flow45
@@ -2238,34 +2235,33 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB19_16
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
-; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
-; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v4, v0, v5
+; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v5, v1, v5
; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v2, v5, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v3, v5, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v6, v5, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v7, v5, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v5, -16, v5
; CHECK-NEXT: s_waitcnt vmcnt(3)
-; CHECK-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
-; CHECK-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:8
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dword v6, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v6, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v7, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v7, v4, s[0:3], 0 offen
; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
-; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
+; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
; CHECK-NEXT: s_or_b32 s6, vcc_lo, s6
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB19_15
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1845bb56be388..2ca1214ba7428 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -7,14 +7,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p0_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -26,12 +24,12 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_2
; GFX942-SDAG-NEXT: .LBB0_3: ; %Flow4
@@ -41,7 +39,6 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
@@ -132,14 +129,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p1_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -151,12 +146,12 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_2
; GFX942-SDAG-NEXT: .LBB1_3: ; %Flow4
@@ -166,7 +161,6 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
@@ -257,48 +251,43 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p3_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v7, v6, v6 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v7, v6, v6 offset1:1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v9, 16, v9
+; GFX942-SDAG-NEXT: v_add_u32_e32 v7, 16, v7
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB2_2
; GFX942-SDAG-NEXT: .LBB2_3: ; %Flow7
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, v0, v2
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB2_5: ; %dynamic-memset-expansion-residual-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
; GFX942-SDAG-NEXT: ds_write_b8 v0, v1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
@@ -376,47 +365,45 @@ define void @memset_p5_varsize_align_4_varsetval(ptr addrspace(5) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p5_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB3_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB3_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v3, v[6:9], off
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v10, v[4:7], off
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v3, 16, v3
+; GFX942-SDAG-NEXT: v_add_u32_e32 v10, 16, v10
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB3_2
; GFX942-SDAG-NEXT: .LBB3_3: ; %Flow7
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB3_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, v0, v2
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB3_5: ; %dynamic-memset-expansion-residual-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
; GFX942-SDAG-NEXT: scratch_store_byte v0, v1, off
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
@@ -494,54 +481,12 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-LABEL: memset_p0_sz1055_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
@@ -552,41 +497,41 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-SDAG-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[38:39], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[10:11], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[20:23] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, 48
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[12:15] offset:32
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[8:11] offset:16
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[60:61], v[38:39], 0, s[20:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7] offset:128
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[32:35]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[28:31]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[32:35]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[28:31]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[20:23]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[12:15]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[60:61], v[8:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[10:11], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[10:11], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[18:19], v[10:11], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[20:21], v[10:11], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[22:23], v[10:11], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[24:25], v[10:11], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[26:27], v[10:11], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[28:29], v[10:11], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[30:31], v[10:11], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[32:33], v[10:11], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[18:19], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[20:21], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[22:23], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[24:25], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[26:27], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[28:29], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[30:31], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[32:33], v[4:7]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB4_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -600,20 +545,6 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
; GFX942-SDAG-NEXT: flat_store_short v[0:1], v3 offset:1052
; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 offset:1054
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -687,52 +618,12 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-LABEL: memset_p0_sz2048_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
@@ -743,55 +634,43 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[36:37], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[8:9], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[18:21] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, 48
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[10:13] offset:32
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[6:9] offset:16
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[36:37], 0, s[20:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5] offset:128
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[10:13]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[6:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[8:9], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[8:9], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[8:9], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[18:19], v[8:9], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[20:21], v[8:9], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[22:23], v[8:9], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[24:25], v[8:9], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[26:27], v[8:9], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[28:29], v[8:9], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[30:31], v[8:9], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[18:19], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[20:21], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[22:23], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[24:25], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[26:27], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[28:29], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[30:31], v[2:5]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB5_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -849,62 +728,34 @@ define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-SDAG-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:112
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:96
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:80
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:64
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:240
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:224
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:208
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:192
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:176
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:160
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:144
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off offset:128
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:128
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB6_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -993,62 +844,34 @@ define void @memset_p1_sz2048_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:112
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:96
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:80
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:64
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:240
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:224
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:208
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:192
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:176
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:160
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:144
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off offset:128
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:128
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB7_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -1107,79 +930,48 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v0
; GFX942-SDAG-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset0:32 offset1:33
-; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_add_u32_e32 v5, 0x100, v5
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB8_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -1268,77 +1060,46 @@ define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v1, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x800
; GFX942-SDAG-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:32 offset1:33
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB9_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
@@ -1396,62 +1157,34 @@ define void @memset_p5_sz1055_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v0
; GFX942-SDAG-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:112
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:96
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:80
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:64
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:48
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:32
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:16
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:240
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:224
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:208
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:192
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:176
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:160
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:144
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off offset:128
-; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:16
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:128
+; GFX942-SDAG-NEXT: v_add_u32_e32 v8, 0x100, v8
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB10_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -1541,59 +1274,31 @@ define void @memset_p5_sz2048_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:112
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:96
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:80
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:64
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:16
; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:240
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:224
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:208
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:192
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:176
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:160
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:144
; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:128
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB11_1
@@ -1650,12 +1355,11 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -1666,12 +1370,12 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
@@ -1681,7 +1385,6 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
@@ -1772,28 +1475,27 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v5
; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v5
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[6:9], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
@@ -1803,7 +1505,6 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index 8395587d94185..39a2bc0a950f5 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -742,19 +742,18 @@ define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -2, v8
-; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, -2, v1
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 1, v1
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xccccdddd
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
-; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_add_u32_e32 v10, s16, v0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
@@ -762,10 +761,10 @@ define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 2
; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[8:9]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v10, s[0:3], 0 offen
; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: v_add_u32_e32 v10, 32, v10
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow3
@@ -775,7 +774,7 @@ define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v8
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
; GFX942-SDAG-NEXT: v_add3_u32 v2, v0, v1, s16
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0xccccdddd
@@ -866,18 +865,17 @@ define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 1
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -2, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -2, v1
; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, 1, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, 1, v1
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-SDAG-NEXT: v_add_u32_e32 v6, s16, v0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v8
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v9
; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
@@ -885,10 +883,10 @@ define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 1
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 2
; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[4:5]
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[8:11], v6, s[0:3], 0 offen
; GFX942-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
+; GFX942-SDAG-NEXT: v_add_u32_e32 v6, 32, v6
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow3
@@ -898,7 +896,7 @@ define void @memset_pattern_i64_as7_dynlen_dynval(ptr addrspace(7) inreg align 1
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v4
+; GFX942-SDAG-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX942-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
; GFX942-SDAG-NEXT: v_add3_u32 v0, v0, v1, s16
; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
@@ -984,37 +982,9 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, s16, v0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v1
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v3
-; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v0
; GFX942-SDAG-NEXT: .LBB14_1: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1022,21 +992,21 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:16
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:32
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:48
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:64
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:80
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:96
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:112
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:16
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:32
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:48
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:64
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:80
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:96
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:112
; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:128
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:144
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:160
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:176
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:192
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:208
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:224
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:240
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:144
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:160
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:176
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:192
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:208
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:224
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:240
; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 0x2000, v1
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB14_1
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
index cff5c2b8fd0f5..585c7cf597c8f 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-fold-reg-sequence-subreg.mir
@@ -162,7 +162,7 @@ body: |
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
; CHECK-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY4]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
index ddb38d336a9ca..4b6f561d9ba4d 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
@@ -48,7 +48,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]].sub0_sub1, %subreg.sub2_sub3, [[COPY1]].sub2_sub3, %subreg.sub0_sub1
; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]].sub1_sub2_sub3, %subreg.sub0_sub1_sub2, [[COPY1]].sub0, %subreg.sub3
- ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]].sub1, %subreg.sub0, [[REG_SEQUENCE]].sub2, %subreg.sub1, [[REG_SEQUENCE]].sub3, %subreg.sub2, [[COPY1]].sub0, %subreg.sub3
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]].sub3, %subreg.sub0, [[COPY]].sub0, %subreg.sub1, [[COPY]].sub1, %subreg.sub2, [[COPY1]].sub0, %subreg.sub3
; GCN-NEXT: KILL implicit [[REG_SEQUENCE2]]
%0:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -113,7 +113,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GCN-NEXT: $vgpr0 = COPY [[COPY3]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
@@ -135,7 +135,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY [[REG_SEQUENCE]]
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GCN-NEXT: $vgpr0 = COPY [[COPY3]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
@@ -203,8 +203,8 @@ body: |
; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub1_sub2
; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
- ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GCN-NEXT: $vgpr0_vgpr1 = COPY [[COPY4]]
; GCN-NEXT: $vgpr2_vgpr3 = COPY [[COPY5]]
; GCN-NEXT: $vgpr4_vgpr5 = COPY [[COPY6]]
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 5b232348c4577..b113380874cd3 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -377,8 +377,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX8-NEXT: v_mov_b32_e32 v1, s35
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s34, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x2800, v0
; GFX8-NEXT: s_movk_i32 s0, 0x2800
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v6, 0
; GFX8-NEXT: v_mov_b32_e32 v7, 0
@@ -723,92 +723,92 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: s_swappc_b64 s[30:31], s[4:5]
; GFX90A-NEXT: v_and_b32_e32 v1, 0xff, v0
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 17, v0
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xfe000000, v0
-; GFX90A-NEXT: v_lshl_or_b32 v1, v1, 3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, s35
-; GFX90A-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
-; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x2800, v1
-; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], 0, 0
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xfe000000, v0
+; GFX90A-NEXT: v_lshl_or_b32 v0, v1, 3, v6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s35
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s34, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x2800, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], 0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7f
; GFX90A-NEXT: s_movk_i32 s2, 0xf000
; GFX90A-NEXT: s_movk_i32 s3, 0x1000
; GFX90A-NEXT: s_movk_i32 s4, 0x2000
; GFX90A-NEXT: .LBB1_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB1_2 Depth 2
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_mov_b32 s5, 0
; GFX90A-NEXT: .LBB1_2: ; %for.body
; GFX90A-NEXT: ; Parent Loop BB1_1 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT: v_add_co_u32_e64 v18, s[0:1], s3, v6
-; GFX90A-NEXT: v_addc_co_u32_e64 v19, s[0:1], 0, v7, s[0:1]
-; GFX90A-NEXT: v_add_co_u32_e64 v20, s[0:1], s4, v6
-; GFX90A-NEXT: v_add_co_u32_e32 v8, vcc, 0xffffe000, v6
-; GFX90A-NEXT: v_addc_co_u32_e64 v21, s[0:1], 0, v7, s[0:1]
-; GFX90A-NEXT: v_addc_co_u32_e32 v9, vcc, -1, v7, vcc
+; GFX90A-NEXT: v_add_co_u32_e64 v18, s[0:1], s3, v4
+; GFX90A-NEXT: v_addc_co_u32_e64 v19, s[0:1], 0, v5, s[0:1]
+; GFX90A-NEXT: v_add_co_u32_e64 v20, s[0:1], s4, v4
+; GFX90A-NEXT: v_add_co_u32_e32 v8, vcc, 0xffffe000, v4
+; GFX90A-NEXT: v_addc_co_u32_e64 v21, s[0:1], 0, v5, s[0:1]
+; GFX90A-NEXT: v_addc_co_u32_e32 v9, vcc, -1, v5, vcc
; GFX90A-NEXT: global_load_dwordx2 v[24:25], v[20:21], off offset:-4096
; GFX90A-NEXT: global_load_dwordx2 v[26:27], v[20:21], off
; GFX90A-NEXT: global_load_dwordx2 v[28:29], v[8:9], off offset:-2048
; GFX90A-NEXT: global_load_dwordx2 v[30:31], v[8:9], off
-; GFX90A-NEXT: v_add_co_u32_e32 v22, vcc, s2, v6
-; GFX90A-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v7, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v22, vcc, s2, v4
+; GFX90A-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v5, vcc
; GFX90A-NEXT: global_load_dwordx2 v[8:9], v[22:23], off offset:-2048
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: global_load_dwordx2 v[18:19], v[18:19], off offset:2048
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: global_load_dwordx2 v[20:21], v[20:21], off offset:2048
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: global_load_dwordx2 v[10:11], v[6:7], off offset:-4096
-; GFX90A-NEXT: global_load_dwordx2 v[12:13], v[6:7], off offset:-2048
-; GFX90A-NEXT: global_load_dwordx2 v[14:15], v[6:7], off
-; GFX90A-NEXT: global_load_dwordx2 v[16:17], v[6:7], off offset:2048
-; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x10000, v6
-; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX90A-NEXT: global_load_dwordx2 v[10:11], v[4:5], off offset:-4096
+; GFX90A-NEXT: global_load_dwordx2 v[12:13], v[4:5], off offset:-2048
+; GFX90A-NEXT: global_load_dwordx2 v[14:15], v[4:5], off
+; GFX90A-NEXT: global_load_dwordx2 v[16:17], v[4:5], off offset:2048
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x10000, v4
+; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX90A-NEXT: s_addk_i32 s5, 0x2000
; GFX90A-NEXT: s_cmp_gt_u32 s5, 0x3fffff
; GFX90A-NEXT: s_waitcnt vmcnt(8)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v28, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v29, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v28, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v29, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(7)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v30, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v31, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v30, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v31, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(6)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v8, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v9, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v8, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v9, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(3)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v10, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v11, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v10, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v11, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(2)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v12, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v13, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v12, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v13, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v14, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v15, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v14, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v15, v3, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v16, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v17, v5, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v24, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v25, v5, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v18, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v19, v5, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v26, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v27, v5, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v20, v4
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v21, v5, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v16, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v17, v3, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v24, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v25, v3, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v18, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v19, v3, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v26, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v27, v3, vcc
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v20, v2
+; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v21, v3, vcc
; GFX90A-NEXT: s_cbranch_scc0 .LBB1_2
; GFX90A-NEXT: ; %bb.3: ; %while.cond.loopexit
; GFX90A-NEXT: ; in Loop: Header=BB1_1 Depth=1
-; GFX90A-NEXT: v_subrev_co_u32_e32 v1, vcc, 1, v1
+; GFX90A-NEXT: v_subrev_co_u32_e32 v7, vcc, 1, v7
; GFX90A-NEXT: s_and_b64 vcc, exec, vcc
; GFX90A-NEXT: s_cbranch_vccz .LBB1_1
; GFX90A-NEXT: ; %bb.4: ; %while.end
; GFX90A-NEXT: v_mov_b32_e32 v1, s35
-; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s34, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, s34, v6
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[4:5], off
+; GFX90A-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX90A-NEXT: s_endpgm
;
; GFX11-LABEL: clmem_read:
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index db330830f6e44..52def643e377e 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -59,7 +59,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, 64, v8
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v23, v20
; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v6, v7
@@ -89,10 +88,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v8
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v9, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v10, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v11, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v23, vcc, 1, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v10, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v11, vcc
; GFX9-NEXT: v_sub_u32_e32 v11, 0x7f, v8
; GFX9-NEXT: v_sub_u32_e32 v9, 64, v11
; GFX9-NEXT: v_lshlrev_b64 v[6:7], v11, v[2:3]
@@ -117,26 +116,26 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
-; GFX9-NEXT: v_sub_u32_e32 v12, 64, v24
-; GFX9-NEXT: v_lshrrev_b64 v[10:11], v24, v[0:1]
+; GFX9-NEXT: v_sub_u32_e32 v12, 64, v23
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], v23, v[0:1]
; GFX9-NEXT: v_lshlrev_b64 v[12:13], v12, v[2:3]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
; GFX9-NEXT: v_or_b32_e32 v12, v10, v12
-; GFX9-NEXT: v_subrev_u32_e32 v10, 64, v24
+; GFX9-NEXT: v_subrev_u32_e32 v10, 64, v23
; GFX9-NEXT: v_or_b32_e32 v13, v11, v13
; GFX9-NEXT: v_lshrrev_b64 v[10:11], v10, v[2:3]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
; GFX9-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v15, v11, v1, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v10, v12, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[10:11], v24, v[2:3]
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], v23, v[2:3]
; GFX9-NEXT: v_cndmask_b32_e64 v14, v12, v0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v11, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v28, vcc, -1, v22
-; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v21, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v4, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v31, vcc, -1, v5, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v27, vcc, -1, v22
+; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v21, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v4, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v5, vcc
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
@@ -145,39 +144,39 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_mov_b32_e32 v11, 0
; GFX9-NEXT: .LBB0_3: ; %udiv-do-while
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_lshlrev_b64 v[32:33], 1, v[8:9]
+; GFX9-NEXT: v_lshlrev_b64 v[31:32], 1, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17]
-; GFX9-NEXT: v_or_b32_e32 v8, v18, v32
+; GFX9-NEXT: v_or_b32_e32 v8, v18, v31
; GFX9-NEXT: v_lshrrev_b32_e32 v18, 31, v15
; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[14:15]
; GFX9-NEXT: v_or_b32_e32 v16, v16, v18
; GFX9-NEXT: v_lshrrev_b32_e32 v18, 31, v7
; GFX9-NEXT: v_or_b32_e32 v14, v14, v18
-; GFX9-NEXT: v_sub_co_u32_e32 v18, vcc, v28, v14
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v29, v15, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v30, v16, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v31, v17, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v32, 31, v18
-; GFX9-NEXT: v_and_b32_e32 v18, v32, v22
+; GFX9-NEXT: v_sub_co_u32_e32 v18, vcc, v27, v14
+; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v28, v15, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v29, v16, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v30, v17, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v31, 31, v18
+; GFX9-NEXT: v_and_b32_e32 v18, v31, v22
; GFX9-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; GFX9-NEXT: v_or_b32_e32 v9, v19, v33
-; GFX9-NEXT: v_and_b32_e32 v19, v32, v21
+; GFX9-NEXT: v_or_b32_e32 v9, v19, v32
+; GFX9-NEXT: v_and_b32_e32 v19, v31, v21
; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18
; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc
-; GFX9-NEXT: v_and_b32_e32 v18, v32, v4
-; GFX9-NEXT: v_and_b32_e32 v19, v32, v5
+; GFX9-NEXT: v_and_b32_e32 v18, v31, v4
+; GFX9-NEXT: v_and_b32_e32 v19, v31, v5
; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v16, v18, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v19, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, -1, v24
+; GFX9-NEXT: v_add_co_u32_e32 v23, vcc, -1, v23
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v26, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v27, vcc
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; GFX9-NEXT: v_or_b32_e32 v18, v24, v26
-; GFX9-NEXT: v_or_b32_e32 v19, v25, v27
+; GFX9-NEXT: v_or_b32_e32 v18, v23, v25
+; GFX9-NEXT: v_or_b32_e32 v19, v24, v26
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
; GFX9-NEXT: v_or3_b32 v6, v6, v10, v12
-; GFX9-NEXT: v_and_b32_e32 v10, 1, v32
+; GFX9-NEXT: v_and_b32_e32 v10, 1, v31
; GFX9-NEXT: v_or3_b32 v7, v7, 0, v13
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v19, v11
@@ -219,13 +218,13 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc
; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v23
+; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20
; GFX9-NEXT: v_xor_b32_e32 v2, v2, v20
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v23, vcc
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v23
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v20, vcc
+; GFX9-NEXT: v_xor_b32_e32 v3, v3, v20
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v20, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v23, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v20, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-O0-LABEL: v_srem_i128_vv:
diff --git a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-unused-lanes.ll b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-unused-lanes.ll
index 32b8ed7a19597..9fb6be62a9d66 100644
--- a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-unused-lanes.ll
+++ b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-unused-lanes.ll
@@ -9,53 +9,55 @@ define <7 x i32> @multiple_predecessor_unused_lanes(<7 x i32> %ha, i32 %h.sel) {
; CHECK-LABEL: multiple_predecessor_unused_lanes:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v16, v6
-; CHECK-NEXT: v_and_b32_e32 v6, 3, v7
+; CHECK-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v11, v4
+; CHECK-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; CHECK-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_and_b32 v0, 3, v8
; CHECK-NEXT: s_mov_b32 s0, exec_lo
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_cmpx_lt_i32_e32 0, v6
+; CHECK-NEXT: v_cmpx_lt_i32_e32 0, v0
; CHECK-NEXT: s_xor_b32 s0, exec_lo, s0
; CHECK-NEXT: s_cbranch_execz .LBB0_6
; CHECK-NEXT: ; %bb.1: ; %LeafBlock
; CHECK-NEXT: s_mov_b32 s1, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u32_e32 1, v6
+; CHECK-NEXT: v_cmpx_ne_u32_e32 1, v0
; CHECK-NEXT: s_xor_b32 s1, exec_lo, s1
; CHECK-NEXT: ; %bb.2: ; %h.default
-; CHECK-NEXT: v_xor_b32_e32 v5, 1, v5
-; CHECK-NEXT: v_mov_b32_e32 v22, v6
-; CHECK-NEXT: v_dual_mov_b32 v20, v4 :: v_dual_mov_b32 v19, v3
-; CHECK-NEXT: v_dual_mov_b32 v18, v2 :: v_dual_mov_b32 v17, v1
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; CHECK-NEXT: v_dual_mov_b32 v21, v5 :: v_dual_mov_b32 v16, v0
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; CHECK-NEXT: v_xor_b32_e32 v12, 1, v5
; CHECK-NEXT: ; %bb.3: ; %Flow
; CHECK-NEXT: s_and_not1_saveexec_b32 s1, s1
+; CHECK-NEXT: s_cbranch_execz .LBB0_5
; CHECK-NEXT: ; %bb.4: ; %h.shuffle
-; CHECK-NEXT: v_mov_b32_e32 v17, v0
-; CHECK-NEXT: v_mov_b32_e32 v18, v0
-; CHECK-NEXT: v_mov_b32_e32 v20, v0
-; CHECK-NEXT: ; %bb.5: ; %Flow1
+; CHECK-NEXT: v_mov_b32_e32 v8, v7
+; CHECK-NEXT: v_mov_b32_e32 v10, v7
+; CHECK-NEXT: v_dual_mov_b32 v13, v12 :: v_dual_mov_b32 v12, v11
+; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; CHECK-NEXT: v_mov_b32_e32 v11, v10
+; CHECK-NEXT: v_mov_b32_e32 v10, v9
+; CHECK-NEXT: v_mov_b32_e32 v9, v8
+; CHECK-NEXT: v_mov_b32_e32 v8, v7
+; CHECK-NEXT: v_mov_b32_e32 v7, v6
+; CHECK-NEXT: .LBB0_5: ; %Flow1
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; CHECK-NEXT: ; implicit-def: $vgpr5
; CHECK-NEXT: ; implicit-def: $vgpr1
; CHECK-NEXT: .LBB0_6: ; %Flow2
; CHECK-NEXT: s_and_not1_saveexec_b32 s0, s0
; CHECK-NEXT: s_cbranch_execz .LBB0_8
; CHECK-NEXT: ; %bb.7: ; %h.add
-; CHECK-NEXT: v_or_b32_e32 v22, 1, v16
-; CHECK-NEXT: v_or_b32_e32 v21, 1, v5
-; CHECK-NEXT: v_or_b32_e32 v20, 1, v4
-; CHECK-NEXT: v_or_b32_e32 v19, 1, v3
-; CHECK-NEXT: v_or_b32_e32 v18, 1, v2
-; CHECK-NEXT: v_or_b32_e32 v17, 1, v1
-; CHECK-NEXT: v_or_b32_e32 v16, 1, v0
+; CHECK-NEXT: v_or_b32_e32 v13, 1, v6
+; CHECK-NEXT: v_or_b32_e32 v12, 1, v5
+; CHECK-NEXT: v_or_b32_e32 v11, 1, v11
+; CHECK-NEXT: v_or_b32_e32 v10, 1, v10
+; CHECK-NEXT: v_or_b32_e32 v9, 1, v9
+; CHECK-NEXT: v_or_b32_e32 v8, 1, v1
+; CHECK-NEXT: v_or_b32_e32 v7, 1, v7
; CHECK-NEXT: .LBB0_8: ; %UnifiedReturnBlock
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v17
-; CHECK-NEXT: v_dual_mov_b32 v2, v18 :: v_dual_mov_b32 v3, v19
-; CHECK-NEXT: v_dual_mov_b32 v4, v20 :: v_dual_mov_b32 v5, v21
-; CHECK-NEXT: v_mov_b32_e32 v6, v22
+; CHECK-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
+; CHECK-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
+; CHECK-NEXT: v_dual_mov_b32 v4, v11 :: v_dual_mov_b32 v5, v12
+; CHECK-NEXT: v_mov_b32_e32 v6, v13
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
%h.case = and i32 %h.sel, 3
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 38f93f4df18da..9018259618f82 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -409,15 +409,13 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT: v_mov_b32_e32 v12, v10
-; GCN-IR-NEXT: v_mov_b32_e32 v13, v11
; GCN-IR-NEXT: v_cndmask_b32_e64 v5, v7, 0, s[4:5]
; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v6, 0, s[4:5]
; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GCN-IR-NEXT: s_cbranch_execz .LBB1_6
; GCN-IR-NEXT: ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, 1, v2
+; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v2
; GCN-IR-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GCN-IR-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v2
; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[6:7], v2
@@ -428,12 +426,12 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB1_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[6:7], v14
-; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, -1, v0
-; GCN-IR-NEXT: v_addc_u32_e32 v15, vcc, -1, v1, vcc
+; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[6:7], v12
+; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, -1, v0
+; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, -1, v1, vcc
; GCN-IR-NEXT: v_not_b32_e32 v4, v8
-; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, v4, v9
-; GCN-IR-NEXT: v_addc_u32_e64 v17, s[8:9], -1, 0, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, v4, v9
+; GCN-IR-NEXT: v_addc_u32_e64 v15, s[8:9], -1, 0, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
@@ -444,8 +442,8 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3
; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4
; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v14, v6
-; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v15, v7, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v12, v6
+; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v13, v7, vcc
; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2
; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4
; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3
@@ -454,8 +452,8 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_and_b32_e32 v8, v8, v0
; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GCN-IR-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, 1, v16
-; GCN-IR-NEXT: v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, 1, v14
+; GCN-IR-NEXT: v_addc_u32_e32 v15, vcc, 0, v15, vcc
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v9, v5
; GCN-IR-NEXT: v_mov_b32_e32 v8, v4
@@ -470,11 +468,10 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_or_b32_e32 v4, v4, v0
; GCN-IR-NEXT: .LBB1_6: ; %Flow5
; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT: v_xor_b32_e32 v0, v11, v10
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v13, v12
-; GCN-IR-NEXT: v_xor_b32_e32 v3, v4, v0
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v11, v10
+; GCN-IR-NEXT: v_xor_b32_e32 v0, v4, v1
; GCN-IR-NEXT: v_xor_b32_e32 v2, v5, v1
-; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v3, v0
+; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 %x, %y
@@ -1635,12 +1632,12 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
; GCN-IR-NEXT: s_or_b64 s[8:9], s[6:7], s[2:3]
; GCN-IR-NEXT: .LBB11_5: ; %udiv-end
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[8:9], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s4, s6, s4
-; GCN-IR-NEXT: s_subb_u32 s5, s7, s5
+; GCN-IR-NEXT: s_sub_u32 s6, s6, s4
+; GCN-IR-NEXT: s_subb_u32 s7, s7, s4
; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
; GCN-IR-NEXT: s_mov_b32 s2, -1
-; GCN-IR-NEXT: v_mov_b32_e32 v0, s4
-; GCN-IR-NEXT: v_mov_b32_e32 v1, s5
+; GCN-IR-NEXT: v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT: v_mov_b32_e32 v1, s7
; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%result = sdiv i64 24, %x
@@ -1776,7 +1773,6 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GCN-IR-NEXT: v_cndmask_b32_e64 v4, 24, 0, s[4:5]
; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT: v_mov_b32_e32 v11, v10
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
@@ -1793,11 +1789,11 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB12_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, -1, v0
-; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT: v_sub_i32_e32 v14, vcc, 58, v8
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, -1, v0
+; GCN-IR-NEXT: v_addc_u32_e32 v12, vcc, -1, v1, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v13, vcc, 58, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], 24, v6
-; GCN-IR-NEXT: v_subb_u32_e64 v15, s[8:9], 0, 0, vcc
+; GCN-IR-NEXT: v_subb_u32_e64 v14, s[8:9], 0, 0, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
@@ -1808,8 +1804,8 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3
; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4
; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v12, v6
-; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v13, v7, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v11, v6
+; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v12, v7, vcc
; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2
; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4
; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3
@@ -1818,8 +1814,8 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v8, v8, v0
; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GCN-IR-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, 1, v14
-; GCN-IR-NEXT: v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v13, vcc, 1, v13
+; GCN-IR-NEXT: v_addc_u32_e32 v14, vcc, 0, v14, vcc
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v9, v5
; GCN-IR-NEXT: v_mov_b32_e32 v8, v4
@@ -1835,9 +1831,9 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
; GCN-IR-NEXT: .LBB12_6: ; %Flow5
; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN-IR-NEXT: v_xor_b32_e32 v0, v4, v10
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v5, v11
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v5, v10
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 24, %x
ret i64 %result
@@ -1969,7 +1965,6 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GCN-IR-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[4:5]
; GCN-IR-NEXT: s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT: v_mov_b32_e32 v11, v10
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GCN-IR-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
@@ -1987,11 +1982,11 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[10:11]
; GCN-IR-NEXT: s_cbranch_execz .LBB13_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, -1, v0
-; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT: v_sub_i32_e32 v14, vcc, 47, v8
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, -1, v0
+; GCN-IR-NEXT: v_addc_u32_e32 v12, vcc, -1, v1, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v13, vcc, 47, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], s[8:9], v6
-; GCN-IR-NEXT: v_subb_u32_e64 v15, s[8:9], 0, 0, vcc
+; GCN-IR-NEXT: v_subb_u32_e64 v14, s[8:9], 0, 0, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
@@ -2002,8 +1997,8 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_lshrrev_b32_e32 v4, 31, v3
; GCN-IR-NEXT: v_or_b32_e32 v6, v6, v4
; GCN-IR-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v12, v6
-; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v13, v7, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v11, v6
+; GCN-IR-NEXT: v_subb_u32_e32 v4, vcc, v12, v7, vcc
; GCN-IR-NEXT: v_or_b32_e32 v2, v8, v2
; GCN-IR-NEXT: v_ashrrev_i32_e32 v8, 31, v4
; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3
@@ -2012,8 +2007,8 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v8, v8, v0
; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GCN-IR-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, 1, v14
-; GCN-IR-NEXT: v_addc_u32_e32 v15, vcc, 0, v15, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v13, vcc, 1, v13
+; GCN-IR-NEXT: v_addc_u32_e32 v14, vcc, 0, v14, vcc
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v9, v5
; GCN-IR-NEXT: v_mov_b32_e32 v8, v4
@@ -2029,9 +2024,9 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: .LBB13_6: ; %Flow5
; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN-IR-NEXT: v_xor_b32_e32 v0, v4, v10
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v5, v11
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v5, v10
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 32768, %x
ret i64 %result
@@ -2064,7 +2059,6 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[0:1]
-; GCN-IR-NEXT: v_mov_b32_e32 v9, v8
; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[0:1]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
@@ -2085,9 +2079,9 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB14_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6
+; GCN-IR-NEXT: v_add_i32_e32 v9, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7
-; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc
+; GCN-IR-NEXT: v_addc_u32_e64 v10, s[8:9], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
@@ -2107,8 +2101,8 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6
; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 1, v10
-; GCN-IR-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v9, vcc, 1, v9
+; GCN-IR-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v7, v3
@@ -2125,9 +2119,9 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: .LBB14_6: ; %Flow5
; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN-IR-NEXT: v_xor_b32_e32 v0, v2, v8
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v9
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v8
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = sdiv i64 %x, 32768
ret i64 %result
@@ -2464,7 +2458,6 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[0:1]
-; GCN-IR-NEXT: v_mov_b32_e32 v9, v8
; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[0:1]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
@@ -2485,9 +2478,9 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB19_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 0xffffffcf, v6
+; GCN-IR-NEXT: v_add_i32_e32 v9, vcc, 0xffffffcf, v6
; GCN-IR-NEXT: v_lshr_b64 v[4:5], v[4:5], v7
-; GCN-IR-NEXT: v_addc_u32_e64 v11, s[8:9], 0, -1, vcc
+; GCN-IR-NEXT: v_addc_u32_e64 v10, s[8:9], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
@@ -2507,8 +2500,8 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v6, 0x8000, v6
; GCN-IR-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
; GCN-IR-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v10, vcc, 1, v10
-; GCN-IR-NEXT: v_addc_u32_e32 v11, vcc, 0, v11, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v9, vcc, 1, v9
+; GCN-IR-NEXT: v_addc_u32_e32 v10, vcc, 0, v10, vcc
; GCN-IR-NEXT: v_or_b32_e32 v1, v7, v1
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v7, v3
@@ -2525,9 +2518,9 @@ define i64 @v_test_sdiv24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: .LBB19_6: ; %Flow5
; GCN-IR-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN-IR-NEXT: v_xor_b32_e32 v0, v2, v8
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v9
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v3, v8
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v9, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v8, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%x.shr = ashr i64 %x, 40
%result = sdiv i64 %x.shr, 32768
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
index 43409292fec9b..227a4735ada47 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
@@ -30,7 +30,7 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_mov_b32 s24, 0xfefa39ef
; CHECK-NEXT: s_mov_b32 s27, 0x3c7abc9e
; CHECK-NEXT: s_mov_b32 s26, 0x3b39803f
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: s_mov_b32 s28, 0
; CHECK-NEXT: s_mov_b32 s29, 0x7ff00000
; CHECK-NEXT: s_mov_b32 s41, 0x40417e50
@@ -140,7 +140,7 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
; CHECK-NEXT: s_mov_b32 vcc_lo, 0
; CHECK-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v6, v2, 0, s[4:5]
; CHECK-NEXT: s_mov_b64 s[94:95], 0
; CHECK-NEXT: .LBB0_13: ; %inner_stack_loop
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
@@ -177,7 +177,7 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: buffer_load_dword v8, v7, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:4
; CHECK-NEXT: v_cndmask_b32_e64 v11, v5, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, v4, 0, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v10, v6, 0, s[14:15]
; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[28:29]
; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[40:41]
; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[42:43]
@@ -187,8 +187,8 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: v_mul_f64 v[8:9], v[10:11], v[8:9]
; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], 0 offset:4
; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], 0
-; CHECK-NEXT: buffer_store_dword v6, v7, s[0:3], 0 offen
-; CHECK-NEXT: buffer_store_dword v6, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v4, v7, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v4, v7, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_andn2_b64 exec, exec, s[94:95]
; CHECK-NEXT: s_cbranch_execnz .LBB0_13
; CHECK-NEXT: .LBB0_14: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
index c40e35bbe1754..ab946d8667f8c 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
@@ -10,7 +10,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_or_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1632 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
@@ -75,1040 +75,743 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_writelane_b32 v63, s55, 13
; CHECK-NEXT: v_writelane_b32 v63, s30, 14
; CHECK-NEXT: v_writelane_b32 v63, s31, 15
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1264 ; 4-byte Folded Spill
+; CHECK-NEXT: v_mov_b32_e32 v32, v29
+; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1268 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v30, v29
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:368
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:364
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:360
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:356
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:352
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:348
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:344
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:340
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:336
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:332
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:328
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:324
-; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:320
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:316
-; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:312
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:308
-; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:304
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:300
-; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:296
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:292
-; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:288
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:284
-; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:280
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:276
-; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:272
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:268
+; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
+; CHECK-NEXT: v_mov_b32_e32 v32, v27
+; CHECK-NEXT: v_and_b32_e32 v9, 1, v9
+; CHECK-NEXT: v_and_b32_e32 v8, 1, v8
+; CHECK-NEXT: v_and_b32_e32 v6, 1, v6
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[78:79], 1, v9
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[88:89], 1, v8
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[92:93], 1, v6
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:344
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:340
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:336
+; CHECK-NEXT: v_accvgpr_write_b32 a36, v24
+; CHECK-NEXT: v_accvgpr_write_b32 a30, v23
+; CHECK-NEXT: v_accvgpr_write_b32 a28, v22
+; CHECK-NEXT: v_accvgpr_write_b32 a26, v21
+; CHECK-NEXT: v_accvgpr_write_b32 a24, v20
+; CHECK-NEXT: v_mov_b32_e32 v20, v19
+; CHECK-NEXT: v_and_b32_e32 v2, 1, v2
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v2
; CHECK-NEXT: v_and_b32_e32 v17, 1, v17
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v17
; CHECK-NEXT: v_and_b32_e32 v16, 1, v16
+; CHECK-NEXT: v_and_b32_e32 v15, 1, v15
+; CHECK-NEXT: v_and_b32_e32 v14, 1, v14
+; CHECK-NEXT: v_and_b32_e32 v13, 1, v13
+; CHECK-NEXT: v_and_b32_e32 v12, 1, v12
+; CHECK-NEXT: v_and_b32_e32 v11, 1, v11
+; CHECK-NEXT: v_and_b32_e32 v10, 1, v10
+; CHECK-NEXT: v_and_b32_e32 v5, 1, v5
+; CHECK-NEXT: v_and_b32_e32 v4, 1, v4
+; CHECK-NEXT: v_and_b32_e32 v3, 1, v3
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v17
; CHECK-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v16
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v15
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v14
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v13
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[72:73], 1, v12
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[74:75], 1, v11
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[76:77], 1, v10
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[94:95], 1, v5
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v4
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v3
+; CHECK-NEXT: v_and_b32_e32 v7, 1, v7
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[90:91], 1, v7
; CHECK-NEXT: s_mov_b64 s[48:49], 0
; CHECK-NEXT: s_mov_b64 s[52:53], 0
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:1496 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:1500 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v30, v27
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v25
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v23
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; CHECK-NEXT: v_mov_b32_e32 v32, v25
+; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v26, v21
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:136
+; CHECK-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:132
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v20, v19
-; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:136
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:132
-; CHECK-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:128
-; CHECK-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:124
-; CHECK-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:120
-; CHECK-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:116
-; CHECK-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:112
-; CHECK-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:108
-; CHECK-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:104
-; CHECK-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:100
-; CHECK-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:96
-; CHECK-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:92
-; CHECK-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:88
-; CHECK-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
-; CHECK-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:80
-; CHECK-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:76
+; CHECK-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:128
+; CHECK-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:124
+; CHECK-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; CHECK-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:116
+; CHECK-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:112
+; CHECK-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:108
+; CHECK-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:104
+; CHECK-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:100
+; CHECK-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:96
+; CHECK-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:92
+; CHECK-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:88
+; CHECK-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:84
+; CHECK-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:80
+; CHECK-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:76
; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v15
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v14
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v13
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v12
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[72:73], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v11
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[74:75], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v10
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[76:77], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v9
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[78:79], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v8
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[88:89], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v7
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[90:91], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v6
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[92:93], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v5
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[94:95], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v4
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v3
+; CHECK-NEXT: v_and_b32_e32 v19, 1, v19
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v19
+; CHECK-NEXT: buffer_load_ubyte v19, off, s[0:3], s32 offset:792
; CHECK-NEXT: s_waitcnt vmcnt(14)
-; CHECK-NEXT: v_and_b32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0
-; CHECK-NEXT: v_and_b32_e32 v0, 1, v2
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v1
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v0
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:392
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:388
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:384
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:380
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:376
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:372
-; CHECK-NEXT: buffer_load_ubyte v1, off, s[0:3], s32 offset:792
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788
-; CHECK-NEXT: s_waitcnt vmcnt(21)
-; CHECK-NEXT: v_and_b32_e32 v17, 1, v18
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_and_b32_e32 v18, 1, v19
-; CHECK-NEXT: s_waitcnt vmcnt(19)
-; CHECK-NEXT: v_and_b32_e32 v19, 1, v20
-; CHECK-NEXT: s_waitcnt vmcnt(18)
-; CHECK-NEXT: v_and_b32_e32 v20, 1, v21
-; CHECK-NEXT: s_waitcnt vmcnt(17)
-; CHECK-NEXT: v_and_b32_e32 v21, 1, v23
-; CHECK-NEXT: s_waitcnt vmcnt(16)
-; CHECK-NEXT: v_and_b32_e32 v23, 1, v32
+; CHECK-NEXT: v_and_b32_e32 v32, 1, v32
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:332
; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_and_b32_e32 v32, 1, v25
-; CHECK-NEXT: s_waitcnt vmcnt(14)
-; CHECK-NEXT: v_and_b32_e32 v25, 1, v33
-; CHECK-NEXT: s_waitcnt vmcnt(13)
-; CHECK-NEXT: v_and_b32_e32 v33, 1, v27
+; CHECK-NEXT: v_and_b32_e32 v33, 1, v33
; CHECK-NEXT: s_waitcnt vmcnt(12)
-; CHECK-NEXT: v_and_b32_e32 v27, 1, v34
+; CHECK-NEXT: v_and_b32_e32 v36, 1, v36
; CHECK-NEXT: s_waitcnt vmcnt(11)
-; CHECK-NEXT: v_and_b32_e32 v34, 1, v26
-; CHECK-NEXT: s_waitcnt vmcnt(10)
-; CHECK-NEXT: v_and_b32_e32 v26, 1, v29
+; CHECK-NEXT: v_and_b32_e32 v37, 1, v37
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v32
; CHECK-NEXT: s_waitcnt vmcnt(9)
-; CHECK-NEXT: v_and_b32_e32 v29, 1, v31
+; CHECK-NEXT: v_and_b32_e32 v32, 1, v38
; CHECK-NEXT: s_waitcnt vmcnt(8)
-; CHECK-NEXT: v_and_b32_e32 v31, 1, v35
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v32
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v33
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v34
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v31
-; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:264
-; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:260
-; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:256
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:252
-; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:248
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:244
-; CHECK-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:240
-; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:236
-; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:232
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:228
-; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:224
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:220
-; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:216
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:212
-; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:208
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:204
-; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:200
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:196
-; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:192
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:188
-; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:184
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:180
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:176
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:172
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:168
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:164
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:160
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:156
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:152
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:148
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:144
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:140
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v18
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v20
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v17
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v19
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v21
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v23
+; CHECK-NEXT: v_and_b32_e32 v29, 1, v29
+; CHECK-NEXT: s_waitcnt vmcnt(7)
+; CHECK-NEXT: v_and_b32_e32 v31, 1, v31
+; CHECK-NEXT: s_waitcnt vmcnt(6)
+; CHECK-NEXT: v_and_b32_e32 v25, 1, v25
+; CHECK-NEXT: s_waitcnt vmcnt(5)
+; CHECK-NEXT: v_and_b32_e32 v24, 1, v24
+; CHECK-NEXT: s_waitcnt vmcnt(4)
+; CHECK-NEXT: v_and_b32_e32 v23, 1, v23
+; CHECK-NEXT: s_waitcnt vmcnt(3)
+; CHECK-NEXT: v_and_b32_e32 v22, 1, v22
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v33
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v36
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v37
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v32
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v29
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v31
; CHECK-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v25
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v27
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v26
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v29
-; CHECK-NEXT: buffer_store_dword a18, off, s[0:3], s32 offset:1136 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword a19, off, s[0:3], s32 offset:1140 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a20, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a21, off, s[0:3], s32 offset:1148 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a22, off, s[0:3], s32 offset:1152 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a23, off, s[0:3], s32 offset:1156 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a24, off, s[0:3], s32 offset:1160 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a25, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a26, off, s[0:3], s32 offset:1168 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a27, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a28, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:1200 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 offset:1220 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a40, off, s[0:3], s32 offset:1224 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
-; CHECK-NEXT: s_waitcnt vmcnt(60)
-; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a47, off, s[0:3], s32 offset:1252 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a48, off, s[0:3], s32 offset:1256 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a49, off, s[0:3], s32 offset:1260 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:712
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:708
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:704
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:700
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:696
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:692
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:688
-; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:684
-; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:680
-; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:676
-; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:672
-; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:668
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:664
-; CHECK-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:660
-; CHECK-NEXT: ; kill: def $agpr1 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr3 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr5 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr7 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr9 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr11 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr13 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr15 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr17 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr19 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr21 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr23 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr25 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr27 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr29 killed $sgpr4 killed $exec
-; CHECK-NEXT: ; kill: def $agpr31 killed $sgpr4 killed $exec
-; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:656
-; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:652
-; CHECK-NEXT: s_waitcnt vmcnt(48)
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1272 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1276 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:1284 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:1292 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:1296 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:1300 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:1304 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:1312 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:1324 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:1328 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
-; CHECK-NEXT: v_and_b32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v1
-; CHECK-NEXT: s_waitcnt vmcnt(32)
-; CHECK-NEXT: buffer_store_dword a0, off, s[0:3], s32 offset:1000 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword a1, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a2, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a3, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a4, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a5, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a6, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a7, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a8, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a9, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a10, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a11, off, s[0:3], s32 offset:1044 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a12, off, s[0:3], s32 offset:1048 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a13, off, s[0:3], s32 offset:1052 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a14, off, s[0:3], s32 offset:1056 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a15, off, s[0:3], s32 offset:1060 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a16, off, s[0:3], s32 offset:1064 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a17, off, s[0:3], s32 offset:1068 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a18, off, s[0:3], s32 offset:1072 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a19, off, s[0:3], s32 offset:1076 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a20, off, s[0:3], s32 offset:1080 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a21, off, s[0:3], s32 offset:1084 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a22, off, s[0:3], s32 offset:1088 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a23, off, s[0:3], s32 offset:1092 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a24, off, s[0:3], s32 offset:1096 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a25, off, s[0:3], s32 offset:1100 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a26, off, s[0:3], s32 offset:1104 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a27, off, s[0:3], s32 offset:1108 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a28, off, s[0:3], s32 offset:1112 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:412
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:408
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:416
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:404
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:420
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:400
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:396
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:424
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:440
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:436
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:444
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:432
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:448
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:428
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:452
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:784
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:780
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:8
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:4
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:456
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:584
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:580
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:576
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:572
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:568
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:564
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:560
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:556
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:552
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:548
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:544
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:540
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:536
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:532
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:528
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Spill
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v24
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v23
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v22
+; CHECK-NEXT: v_and_b32_e32 v27, 1, v27
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v27
+; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:376
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:372
+; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:368
+; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:364
+; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:360
+; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:356
+; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:352
+; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:348
+; CHECK-NEXT: v_and_b32_e32 v21, 1, v21
+; CHECK-NEXT: v_and_b32_e32 v34, 1, v34
+; CHECK-NEXT: v_and_b32_e32 v35, 1, v35
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v21
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v34
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v35
+; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:784
+; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:780
+; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:788
+; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:392
+; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:388
+; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:384
+; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:380
+; CHECK-NEXT: s_waitcnt vmcnt(17)
+; CHECK-NEXT: v_and_b32_e32 v2, 1, v19
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v2
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312
+; CHECK-NEXT: s_waitcnt vmcnt(5)
+; CHECK-NEXT: v_cmp_lt_i32_e64 s[38:39], 2, v21
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1012 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:328
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1024 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:308
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1004 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:324
; CHECK-NEXT: s_waitcnt vmcnt(2)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:1564 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:1568 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:1572 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:1576 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:1580 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:1584 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:1588 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:1592 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:1596 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:1600 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:1604 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:1608 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:1612 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:1616 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:1620 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:1624 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Spill
-; CHECK-NEXT: v_cmp_lt_i32_e64 s[38:39], 2, v0
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1028 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1008 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:320
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1032 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1016 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:316
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1036 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296
+; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1020 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword a15, off, s[0:3], s32 offset:292
+; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:288
+; CHECK-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:284
+; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:280
+; CHECK-NEXT: buffer_load_dword a11, off, s[0:3], s32 offset:276
+; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:272
+; CHECK-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:268
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264
+; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:260
+; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:256
+; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:252
+; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:248
+; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:244
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:240
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:236
+; CHECK-NEXT: buffer_load_dword a13, off, s[0:3], s32 offset:232
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:228
+; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:224
+; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:220
+; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:216
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:212
+; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:208
+; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:204
+; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:200
+; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:196
+; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:192
+; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:188
+; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:184
+; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:180
+; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:176
+; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:172
+; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:168
+; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:164
+; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:160
+; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:156
+; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:152
+; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:148
+; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:144
+; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:140
+; CHECK-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:712
+; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:584
+; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:580
+; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:576
+; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:572
+; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:568
+; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:564
+; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:560
+; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:556
+; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:552
+; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:548
+; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:544
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:540
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:536
+; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:532
+; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:528
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:524
+; CHECK-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:456
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:8
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:4
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32
+; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:708
+; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:704
+; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:700
+; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:696
+; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:692
+; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:688
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:684
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:452
+; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:448
+; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:444
+; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:440
+; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:436
+; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:432
+; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:428
+; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:424
+; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:420
+; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:416
+; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:412
+; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:408
+; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:404
+; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:400
+; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:680
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:676
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:672
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:668
+; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:664
+; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:660
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:656
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:652
+; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:396
+; CHECK-NEXT: s_waitcnt vmcnt(62)
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1040 ; 4-byte Folded Spill
; CHECK-NEXT: s_and_saveexec_b64 s[50:51], s[38:39]
; CHECK-NEXT: s_xor_b64 s[50:51], exec, s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_4
; CHECK-NEXT: ; %bb.1: ; %LeafBlock46
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[38:39], 3, v0
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[38:39], 3, v21
; CHECK-NEXT: s_mov_b64 s[54:55], -1
; CHECK-NEXT: s_and_saveexec_b64 s[52:53], s[38:39]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.2: ; %bb2
-; CHECK-NEXT: v_accvgpr_write_b32 a10, v42
-; CHECK-NEXT: v_accvgpr_write_b32 a11, v43
-; CHECK-NEXT: v_accvgpr_write_b32 a8, v40
-; CHECK-NEXT: v_accvgpr_write_b32 a14, v48
-; CHECK-NEXT: v_accvgpr_write_b32 a15, v49
-; CHECK-NEXT: v_accvgpr_write_b32 a4, v56
-; CHECK-NEXT: v_accvgpr_write_b32 a5, v57
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v50
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v51
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_write_b32 a16, v58
-; CHECK-NEXT: v_accvgpr_write_b32 a17, v59
-; CHECK-NEXT: v_accvgpr_read_b32 v60, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v59, a17
-; CHECK-NEXT: s_xor_b64 s[54:55], exec, -1
+; CHECK-NEXT: v_xor_b32_e32 v2, v28, v28
+; CHECK-NEXT: s_waitcnt vmcnt(31)
+; CHECK-NEXT: v_accvgpr_write_b32 a44, v6
+; CHECK-NEXT: s_waitcnt vmcnt(30)
+; CHECK-NEXT: v_xor_b32_e32 v6, v24, v51
+; CHECK-NEXT: v_xor_b32_e32 v21, v55, v50
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 17, v21
+; CHECK-NEXT: v_accvgpr_write_b32 a1, v60
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v61
+; CHECK-NEXT: v_lshrrev_b64 v[60:61], 14, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v61, v61, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v26, v26
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v8
; CHECK-NEXT: s_waitcnt vmcnt(29)
-; CHECK-NEXT: v_xor_b32_e32 v28, v28, v18
+; CHECK-NEXT: v_xor_b32_e32 v8, v56, v49
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v9
+; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 18, v6
+; CHECK-NEXT: v_mov_b32_e32 v49, v58
+; CHECK-NEXT: v_accvgpr_write_b32 a0, v59
+; CHECK-NEXT: v_lshrrev_b64 v[58:59], 13, v[8:9]
+; CHECK-NEXT: v_or_b32_e32 v59, v59, v2
+; CHECK-NEXT: v_add_co_u32_e64 v50, s[38:39], 24, v34
+; CHECK-NEXT: v_xor_b32_e32 v2, v24, v24
+; CHECK-NEXT: v_addc_co_u32_e64 v51, s[38:39], 0, v35, s[38:39]
; CHECK-NEXT: s_waitcnt vmcnt(28)
-; CHECK-NEXT: v_xor_b32_e32 v0, v29, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a9, v0
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a27
-; CHECK-NEXT: s_waitcnt vmcnt(26)
-; CHECK-NEXT: v_xor_b32_e32 v29, v31, v29
-; CHECK-NEXT: s_waitcnt vmcnt(24)
-; CHECK-NEXT: v_xor_b32_e32 v0, v33, v7
-; CHECK-NEXT: v_accvgpr_write_b32 a27, v0
-; CHECK-NEXT: v_xor_b32_e32 v0, v30, v4
-; CHECK-NEXT: v_accvgpr_write_b32 a12, v0
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_xor_b32_e32 v62, v37, v11
-; CHECK-NEXT: v_xor_b32_e32 v0, v32, v6
-; CHECK-NEXT: v_xor_b32_e32 v32, v34, v8
-; CHECK-NEXT: v_xor_b32_e32 v34, v36, v10
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a46
-; CHECK-NEXT: v_xor_b32_e32 v33, v35, v9
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: v_xor_b32_e32 v35, v39, v13
-; CHECK-NEXT: s_waitcnt vmcnt(15)
-; CHECK-NEXT: v_xor_b32_e32 v10, v44, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a44
-; CHECK-NEXT: s_waitcnt vmcnt(11)
-; CHECK-NEXT: v_xor_b32_e32 v6, v48, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a42
-; CHECK-NEXT: v_xor_b32_e32 v9, v47, v21
-; CHECK-NEXT: s_waitcnt vmcnt(8)
-; CHECK-NEXT: v_xor_b32_e32 v21, v51, v25
-; CHECK-NEXT: v_accvgpr_write_b32 a36, v0
-; CHECK-NEXT: s_waitcnt vmcnt(6)
-; CHECK-NEXT: v_xor_b32_e32 v13, v53, v31
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a29
-; CHECK-NEXT: s_waitcnt vmcnt(4)
-; CHECK-NEXT: v_xor_b32_e32 v1, v55, v31
-; CHECK-NEXT: s_waitcnt vmcnt(3)
-; CHECK-NEXT: v_xor_b32_e32 v25, v56, v60
-; CHECK-NEXT: v_xor_b32_e32 v8, v46, v20
-; CHECK-NEXT: v_xor_b32_e32 v20, v50, v30
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a38
-; CHECK-NEXT: v_xor_b32_e32 v0, v54, v58
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
-; CHECK-NEXT: v_xor_b32_e32 v11, v45, v19
-; CHECK-NEXT: v_xor_b32_e32 v19, v27, v5
-; CHECK-NEXT: v_xor_b32_e32 v4, v52, v30
-; CHECK-NEXT: v_xor_b32_e32 v30, v26, v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v5, 17, v25
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 14, v[0:1]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v5
-; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v13
-; CHECK-NEXT: v_xor_b32_e32 v18, v38, v12
-; CHECK-NEXT: v_lshlrev_b32_e32 v12, 18, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 13, v[4:5]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v12
-; CHECK-NEXT: v_xor_b32_e32 v7, v49, v23
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a37
-; CHECK-NEXT: s_waitcnt vmcnt(2)
-; CHECK-NEXT: v_xor_b32_e32 v23, v57, v31
-; CHECK-NEXT: v_xor_b32_e32 v17, v43, v17
-; CHECK-NEXT: v_xor_b32_e32 v16, v42, v16
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_xor_b32_e32 v15, v41, v15
-; CHECK-NEXT: v_xor_b32_e32 v14, v40, v14
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_accvgpr_read_b32 v57, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v56, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v43, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v42, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v40, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v49, a15
-; CHECK-NEXT: v_accvgpr_read_b32 v48, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v51, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v50, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a16
-; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: v_add_co_u32_e64 v12, s[38:39], 24, v36
+; CHECK-NEXT: v_xor_b32_e32 v6, v46, v39
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v2
+; CHECK-NEXT: flat_store_dwordx4 v[50:51], v[58:61] offset:80
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v8
+; CHECK-NEXT: v_lshrrev_b64 v[60:61], 12, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v61, v61, v2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_addc_co_u32_e64 v13, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:80
+; CHECK-NEXT: v_xor_b32_e32 v2, v22, v22
+; CHECK-NEXT: v_xor_b32_e32 v8, v48, v37
+; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v6
+; CHECK-NEXT: v_lshrrev_b64 v[58:59], 11, v[8:9]
+; CHECK-NEXT: v_or_b32_e32 v59, v59, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v20, v20
+; CHECK-NEXT: v_xor_b32_e32 v6, v22, v33
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v8
+; CHECK-NEXT: v_lshrrev_b64 v[8:9], 10, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v9, v9, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v18, v18
+; CHECK-NEXT: flat_store_dwordx4 v[50:51], v[58:61] offset:64
; CHECK-NEXT: s_movk_i32 s38, 0x7c
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 19, v4
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 12, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 20, v20
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 11, v[6:7]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 22, v8
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 21, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 10, v[8:9]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 9, v[10:11]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 26, v18
-; CHECK-NEXT: v_add_co_u32_e64 v0, s[38:39], s38, v36
-; CHECK-NEXT: v_addc_co_u32_e64 v1, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v23
+; CHECK-NEXT: v_xor_b32_e32 v58, v44, v31
+; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 22, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 9, v[58:59]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v12, v12
+; CHECK-NEXT: flat_store_dwordx4 v[50:51], v[6:9] offset:48
+; CHECK-NEXT: v_xor_b32_e32 v60, v38, v29
+; CHECK-NEXT: v_xor_b32_e32 v6, v40, v25
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v60
+; CHECK-NEXT: v_lshrrev_b64 v[8:9], 6, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v9, v9, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v10, v10
+; CHECK-NEXT: v_mov_b32_e32 v31, v52
+; CHECK-NEXT: v_xor_b32_e32 v52, v32, v23
+; CHECK-NEXT: v_mov_b32_e32 v23, v27
+; CHECK-NEXT: v_mov_b32_e32 v27, v53
+; CHECK-NEXT: v_and_b32_e32 v53, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 26, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 5, v[52:53]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: flat_store_dwordx4 v[50:51], v[6:9] offset:16
+; CHECK-NEXT: v_xor_b32_e32 v2, v30, v30
+; CHECK-NEXT: v_add_co_u32_e64 v6, s[38:39], s38, v34
+; CHECK-NEXT: v_addc_co_u32_e64 v7, s[38:39], 0, v35, s[38:39]
+; CHECK-NEXT: v_mov_b32_e32 v29, v3
+; CHECK-NEXT: v_lshrrev_b32_e32 v3, 15, v2
; CHECK-NEXT: s_movk_i32 s38, 0x78
-; CHECK-NEXT: flat_store_short v[0:1], v2
-; CHECK-NEXT: v_add_co_u32_e64 v0, s[38:39], s38, v36
-; CHECK-NEXT: v_alignbit_b32 v2, v23, v25, 15
-; CHECK-NEXT: v_addc_co_u32_e64 v1, s[38:39], 0, v37, s[38:39]
-; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_and_b32_e32 v0, 0x7fffffff, v19
-; CHECK-NEXT: v_lshlrev_b32_e32 v1, 31, v28
-; CHECK-NEXT: v_or_b32_e32 v31, v0, v1
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v10
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 8, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 24, v16
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 7, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v35
-; CHECK-NEXT: flat_store_dwordx2 v[12:13], v[0:1] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 25, v14
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 6, v[18:19]
-; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v62
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 5, v[34:35]
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v4
-; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[0:3] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a36
-; CHECK-NEXT: v_mov_b32_e32 v0, v33
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 4, v[32:33]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 27, v34
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:32
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a36
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 3, v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 28, v32
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[12:13], v[0:1]
-; CHECK-NEXT: v_mov_b32_e32 v0, v29
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a12
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v6
-; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[36:37], v[0:1] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a12
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v0
-; CHECK-NEXT: v_lshrrev_b64 v[32:33], 1, v[28:29]
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v4
-; CHECK-NEXT: v_or_b32_e32 v33, v33, v0
-; CHECK-NEXT: flat_store_dwordx4 v[36:37], v[30:33]
+; CHECK-NEXT: flat_store_short v[6:7], v3
+; CHECK-NEXT: v_add_co_u32_e64 v6, s[38:39], s38, v34
+; CHECK-NEXT: v_xor_b32_e32 v9, v2, v2
+; CHECK-NEXT: v_alignbit_b32 v2, v2, v21, 15
+; CHECK-NEXT: v_addc_co_u32_e64 v7, s[38:39], 0, v35, s[38:39]
+; CHECK-NEXT: flat_store_dword v[6:7], v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v16, v16
+; CHECK-NEXT: v_xor_b32_e32 v6, v42, v17
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v58
+; CHECK-NEXT: v_lshrrev_b64 v[58:59], 8, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v59, v59, v2
+; CHECK-NEXT: v_xor_b32_e32 v2, v14, v14
+; CHECK-NEXT: v_and_b32_e32 v61, 0x7fffffff, v2
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[58:59] offset:64
+; CHECK-NEXT: v_xor_b32_e32 v58, v16, v15
+; CHECK-NEXT: v_xor_b32_e32 v3, v6, v6
+; CHECK-NEXT: v_lshlrev_b32_e32 v15, 24, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 7, v[60:61]
+; CHECK-NEXT: v_xor_b32_e32 v8, v14, v19
+; CHECK-NEXT: v_xor_b32_e32 v2, v54, v13
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v15
+; CHECK-NEXT: v_xor_b32_e32 v60, v36, v11
+; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CHECK-NEXT: v_xor_b32_e32 v17, v4, v4
+; CHECK-NEXT: flat_store_dwordx2 v[50:51], v[6:7] offset:32
+; CHECK-NEXT: v_xor_b32_e32 v6, v8, v8
+; CHECK-NEXT: v_lshlrev_b32_e32 v11, 27, v52
+; CHECK-NEXT: v_lshrrev_b64 v[52:53], 3, v[2:3]
+; CHECK-NEXT: v_lshlrev_b32_e32 v3, 28, v60
+; CHECK-NEXT: v_and_b32_e32 v61, 0x7fffffff, v6
+; CHECK-NEXT: v_or_b32_e32 v53, v53, v3
+; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v17
+; CHECK-NEXT: v_xor_b32_e32 v13, v0, v0
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 4, v[60:61]
+; CHECK-NEXT: flat_store_dwordx2 v[50:51], v[52:53]
+; CHECK-NEXT: v_lshrrev_b64 v[50:51], 2, v[58:59]
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v2
+; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v11
+; CHECK-NEXT: v_lshlrev_b32_e32 v11, 31, v8
+; CHECK-NEXT: v_or_b32_e32 v51, v51, v2
+; CHECK-NEXT: v_lshrrev_b64 v[8:9], 1, v[8:9]
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 30, v58
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[6:7] offset:32
+; CHECK-NEXT: v_or_b32_e32 v7, v13, v11
+; CHECK-NEXT: v_xor_b32_e32 v6, v4, v5
+; CHECK-NEXT: v_or_b32_e32 v9, v9, v2
+; CHECK-NEXT: v_accvgpr_read_b32 v61, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v60, a1
+; CHECK-NEXT: v_mov_b32_e32 v3, v29
+; CHECK-NEXT: v_mov_b32_e32 v53, v27
+; CHECK-NEXT: v_mov_b32_e32 v52, v31
+; CHECK-NEXT: v_accvgpr_read_b32 v59, a0
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[50:51] offset:16
+; CHECK-NEXT: v_mov_b32_e32 v27, v23
+; CHECK-NEXT: v_mov_b32_e32 v58, v49
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[6:9]
+; CHECK-NEXT: s_xor_b64 s[54:55], exec, -1
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a44
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a3
; CHECK-NEXT: .LBB0_3: ; %Flow52
; CHECK-NEXT: s_or_b64 exec, exec, s[52:53]
; CHECK-NEXT: s_and_b64 s[52:53], s[54:55], exec
-; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr21
; CHECK-NEXT: .LBB0_4: ; %Flow51
; CHECK-NEXT: s_andn2_saveexec_b64 s[50:51], s[50:51]
; CHECK-NEXT: ; %bb.5: ; %LeafBlock
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[38:39], 2, v0
-; CHECK-NEXT: s_andn2_b64 s[48:49], s[52:53], exec
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[38:39], 2, v21
+; CHECK-NEXT: s_andn2_b64 s[52:53], s[52:53], exec
; CHECK-NEXT: s_and_b64 s[38:39], s[38:39], exec
-; CHECK-NEXT: s_or_b64 s[52:53], s[48:49], s[38:39]
; CHECK-NEXT: s_mov_b64 s[48:49], exec
+; CHECK-NEXT: s_or_b64 s[52:53], s[52:53], s[38:39]
; CHECK-NEXT: ; %bb.6: ; %Flow53
; CHECK-NEXT: s_or_b64 exec, exec, s[50:51]
-; CHECK-NEXT: s_mov_b64 s[38:39], exec
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
-; CHECK-NEXT: s_and_b64 s[50:51], s[38:39], s[52:53]
-; CHECK-NEXT: s_mov_b64 exec, s[50:51]
+; CHECK-NEXT: s_and_saveexec_b64 s[38:39], s[52:53]
; CHECK-NEXT: s_cbranch_execz .LBB0_44
; CHECK-NEXT: ; %bb.7: ; %bb0
; CHECK-NEXT: s_and_saveexec_b64 s[50:51], s[46:47]
; CHECK-NEXT: s_xor_b64 s[50:51], exec, s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_41
; CHECK-NEXT: ; %bb.8: ; %bb0b
-; CHECK-NEXT: s_mov_b64 s[52:53], exec
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
-; CHECK-NEXT: s_and_b64 s[36:37], s[52:53], s[36:37]
-; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr9
-; CHECK-NEXT: s_mov_b64 exec, s[36:37]
+; CHECK-NEXT: s_and_saveexec_b64 s[52:53], s[36:37]
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.9: ; %cond.load
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[8:9], v[0:1]
+; CHECK-NEXT: flat_load_dwordx2 v[18:19], v[0:1]
; CHECK-NEXT: .LBB0_10: ; %else
; CHECK-NEXT: s_or_b64 exec, exec, s[52:53]
; CHECK-NEXT: s_mov_b64 s[36:37], exec
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1496 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1500 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1264 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1268 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:996 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
; CHECK-NEXT: s_and_b64 s[34:35], s[36:37], s[34:35]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_accvgpr_read_b32 v14, a24
+; CHECK-NEXT: v_accvgpr_read_b32 v15, a25
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a26
+; CHECK-NEXT: v_accvgpr_read_b32 v17, a27
+; CHECK-NEXT: v_accvgpr_read_b32 v22, a28
+; CHECK-NEXT: v_accvgpr_read_b32 v23, a29
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a30
+; CHECK-NEXT: v_accvgpr_read_b32 v25, a31
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a36
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a37
+; CHECK-NEXT: v_pk_mov_b32 v[38:39], v[26:27], v[26:27] op_sel:[0,1]
; CHECK-NEXT: s_mov_b64 exec, s[34:35]
; CHECK-NEXT: s_cbranch_execz .LBB0_12
; CHECK-NEXT: ; %bb.11: ; %cond.load1
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[0:1] offset:8
+; CHECK-NEXT: flat_load_dwordx2 v[20:21], v[0:1] offset:8
; CHECK-NEXT: .LBB0_12: ; %else2
; CHECK-NEXT: s_or_b64 exec, exec, s[36:37]
; CHECK-NEXT: s_and_saveexec_b64 s[34:35], s[30:31]
; CHECK-NEXT: s_cbranch_execz .LBB0_14
; CHECK-NEXT: ; %bb.13: ; %cond.load4
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[12:13], v[0:1] offset:16
+; CHECK-NEXT: flat_load_dwordx2 v[14:15], v[0:1] offset:16
; CHECK-NEXT: .LBB0_14: ; %else5
; CHECK-NEXT: s_or_b64 exec, exec, s[34:35]
; CHECK-NEXT: s_and_saveexec_b64 s[30:31], s[94:95]
; CHECK-NEXT: s_cbranch_execz .LBB0_16
; CHECK-NEXT: ; %bb.15: ; %cond.load7
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[14:15], v[0:1] offset:24
+; CHECK-NEXT: flat_load_dwordx2 v[16:17], v[0:1] offset:24
; CHECK-NEXT: .LBB0_16: ; %else8
; CHECK-NEXT: s_or_b64 exec, exec, s[30:31]
; CHECK-NEXT: s_and_saveexec_b64 s[94:95], s[92:93]
; CHECK-NEXT: s_cbranch_execz .LBB0_18
; CHECK-NEXT: ; %bb.17: ; %cond.load10
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[22:23], v[0:1] offset:32
; CHECK-NEXT: .LBB0_18: ; %else11
; CHECK-NEXT: s_or_b64 exec, exec, s[94:95]
; CHECK-NEXT: s_and_saveexec_b64 s[92:93], s[90:91]
; CHECK-NEXT: s_cbranch_execz .LBB0_20
; CHECK-NEXT: ; %bb.19: ; %cond.load13
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[16:17], v[0:1] offset:40
+; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[0:1] offset:40
; CHECK-NEXT: .LBB0_20: ; %else14
; CHECK-NEXT: s_or_b64 exec, exec, s[92:93]
; CHECK-NEXT: s_and_saveexec_b64 s[90:91], s[88:89]
; CHECK-NEXT: s_cbranch_execz .LBB0_22
; CHECK-NEXT: ; %bb.21: ; %cond.load16
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[0:1] offset:48
+; CHECK-NEXT: flat_load_dwordx2 v[32:33], v[0:1] offset:48
; CHECK-NEXT: .LBB0_22: ; %else17
; CHECK-NEXT: s_or_b64 exec, exec, s[90:91]
; CHECK-NEXT: s_and_saveexec_b64 s[88:89], s[78:79]
; CHECK-NEXT: s_cbranch_execz .LBB0_24
; CHECK-NEXT: ; %bb.23: ; %cond.load19
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[18:19], v[0:1] offset:56
+; CHECK-NEXT: flat_load_dwordx2 v[36:37], v[0:1] offset:56
; CHECK-NEXT: .LBB0_24: ; %else20
; CHECK-NEXT: s_or_b64 exec, exec, s[88:89]
; CHECK-NEXT: s_and_saveexec_b64 s[78:79], s[76:77]
; CHECK-NEXT: s_cbranch_execz .LBB0_26
; CHECK-NEXT: ; %bb.25: ; %cond.load22
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[26:27], v[0:1] offset:64
+; CHECK-NEXT: flat_load_dwordx2 v[38:39], v[0:1] offset:64
; CHECK-NEXT: .LBB0_26: ; %else23
; CHECK-NEXT: s_or_b64 exec, exec, s[78:79]
; CHECK-NEXT: s_and_saveexec_b64 s[76:77], s[74:75]
; CHECK-NEXT: s_cbranch_execz .LBB0_28
; CHECK-NEXT: ; %bb.27: ; %cond.load25
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[20:21], v[0:1] offset:72
+; CHECK-NEXT: flat_load_dwordx2 v[48:49], v[0:1] offset:72
; CHECK-NEXT: .LBB0_28: ; %else26
; CHECK-NEXT: s_or_b64 exec, exec, s[76:77]
; CHECK-NEXT: s_and_saveexec_b64 s[74:75], s[72:73]
; CHECK-NEXT: s_cbranch_execz .LBB0_30
; CHECK-NEXT: ; %bb.29: ; %cond.load28
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[28:29], v[0:1] offset:80
; CHECK-NEXT: .LBB0_30: ; %else29
; CHECK-NEXT: s_or_b64 exec, exec, s[74:75]
; CHECK-NEXT: s_and_saveexec_b64 s[72:73], s[62:63]
; CHECK-NEXT: s_cbranch_execz .LBB0_32
; CHECK-NEXT: ; %bb.31: ; %cond.load31
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[32:33], v[0:1] offset:88
+; CHECK-NEXT: flat_load_dwordx2 v[50:51], v[0:1] offset:88
; CHECK-NEXT: .LBB0_32: ; %else32
; CHECK-NEXT: s_or_b64 exec, exec, s[72:73]
; CHECK-NEXT: s_and_saveexec_b64 s[62:63], s[60:61]
; CHECK-NEXT: s_cbranch_execz .LBB0_34
; CHECK-NEXT: ; %bb.33: ; %cond.load34
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[30:31], v[0:1] offset:96
; CHECK-NEXT: .LBB0_34: ; %else35
; CHECK-NEXT: s_or_b64 exec, exec, s[62:63]
; CHECK-NEXT: s_and_saveexec_b64 s[60:61], s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_36
; CHECK-NEXT: ; %bb.35: ; %cond.load37
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:104
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
+; CHECK-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:104
; CHECK-NEXT: .LBB0_36: ; %else38
; CHECK-NEXT: s_or_b64 exec, exec, s[60:61]
; CHECK-NEXT: s_and_saveexec_b64 s[58:59], s[56:57]
; CHECK-NEXT: s_cbranch_execz .LBB0_38
; CHECK-NEXT: ; %bb.37: ; %cond.load40
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 a[26:27], v[0:1] offset:112
+; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[0:1] offset:112
; CHECK-NEXT: .LBB0_38: ; %else41
; CHECK-NEXT: s_or_b64 exec, exec, s[58:59]
; CHECK-NEXT: s_and_saveexec_b64 s[56:57], vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_40
; CHECK-NEXT: ; %bb.39: ; %cond.load43
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_load_dwordx2 a[28:29], v[0:1] offset:120
+; CHECK-NEXT: flat_load_dwordx2 v[12:13], v[0:1] offset:120
; CHECK-NEXT: .LBB0_40: ; %else44
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a29
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v5
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v3, a27
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
-; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
-; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v34
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v13
; CHECK-NEXT: flat_store_short v[0:1], v2
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v34
-; CHECK-NEXT: v_alignbit_b32 v2, v5, v4, 15
+; CHECK-NEXT: v_alignbit_b32 v2, v13, v12, 15
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v10
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v9
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a26
-; CHECK-NEXT: v_or_b32_e32 v9, v1, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v4
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
-; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v20
+; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v19
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
+; CHECK-NEXT: v_or_b32_e32 v19, v1, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v12
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 14, v[10:11]
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 18, v10
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 13, v[6:7]
+; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 12, v[30:31]
+; CHECK-NEXT: v_and_b32_e32 v51, 0x7fffffff, v51
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v0
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v34
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[36:37]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v30
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 11, v[50:51]
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v30
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v36
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[30:31]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[32:33]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v32
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[28:29]
+; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v50
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 10, v[28:29]
+; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v49
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 22, v28
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 9, v[48:49]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_and_b32_e32 v39, 0x7fffffff, v39
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v48
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 8, v[38:39]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[4:5] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 24, v38
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 7, v[36:37]
+; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v36
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 6, v[32:33]
+; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5] offset:32
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 26, v32
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 5, v[24:25]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 27, v24
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 4, v[22:23]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v24
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v20
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[26:27]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v26
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[18:19]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v18
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[24:25]
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[4:5] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 28, v22
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 3, v[16:17]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v16
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[22:23]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v22
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v14
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[12:13]
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[10:13] offset:80
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v16
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[14:15]
; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
+; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[0:1] offset:16
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v12
-; CHECK-NEXT: v_or_b32_e32 v11, v11, v0
-; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[8:11]
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v14
+; CHECK-NEXT: v_lshrrev_b64 v[20:21], 1, v[20:21]
+; CHECK-NEXT: v_or_b32_e32 v21, v21, v0
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[18:21]
+; CHECK-NEXT: ; implicit-def: $vgpr55
+; CHECK-NEXT: ; implicit-def: $vgpr24
+; CHECK-NEXT: ; implicit-def: $vgpr56
+; CHECK-NEXT: ; implicit-def: $vgpr46
+; CHECK-NEXT: ; implicit-def: $vgpr48
+; CHECK-NEXT: ; implicit-def: $vgpr22
+; CHECK-NEXT: ; implicit-def: $vgpr44
+; CHECK-NEXT: ; implicit-def: $vgpr42
+; CHECK-NEXT: ; implicit-def: $vgpr38
+; CHECK-NEXT: ; implicit-def: $vgpr40
+; CHECK-NEXT: ; implicit-def: $vgpr32
+; CHECK-NEXT: ; implicit-def: $vgpr36
+; CHECK-NEXT: ; implicit-def: $vgpr54
+; CHECK-NEXT: ; implicit-def: $vgpr16
+; CHECK-NEXT: ; implicit-def: $vgpr14
+; CHECK-NEXT: ; implicit-def: $vgpr4
; CHECK-NEXT: .LBB0_41: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[56:57], s[50:51]
; CHECK-NEXT: s_cbranch_execz .LBB0_43
; CHECK-NEXT: ; %bb.42: ; %bb0a
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
-; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
-; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
-; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
-; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v36
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v52
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
-; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v39
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v34
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
+; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v30
; CHECK-NEXT: flat_store_short v[0:1], v2
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v52
-; CHECK-NEXT: v_alignbit_b32 v2, v39, v38, 15
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v34
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_alignbit_b32 v2, v30, v55, 15
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
; CHECK-NEXT: flat_store_dword v[0:1], v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v10
-; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v9
-; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v37
-; CHECK-NEXT: v_or_b32_e32 v9, v1, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v38
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[36:37]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v0
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v52
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[34:35]
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v34
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[32:33]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[30:31]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v30
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[28:29]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[26:27]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v20
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v26
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[24:25]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v24
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[22:23]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v22
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[20:21]
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[18:19]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v18
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[16:17]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v16
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[14:15]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v14
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[12:13]
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v14
+; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v0
+; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v28
+; CHECK-NEXT: v_or_b32_e32 v5, v1, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v55
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 14, v[24:25]
+; CHECK-NEXT: v_mov_b32_e32 v15, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v57
+; CHECK-NEXT: v_and_b32_e32 v57, 0x7fffffff, v26
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v0
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v34
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 18, v24
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 13, v[56:57]
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
+; CHECK-NEXT: v_mov_b32_e32 v57, v3
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v2
+; CHECK-NEXT: v_mov_b32_e32 v3, v47
+; CHECK-NEXT: v_and_b32_e32 v47, 0x7fffffff, v24
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[10:13] offset:80
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v56
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 12, v[46:47]
+; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v22
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v46
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 11, v[48:49]
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v2
+; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v20
+; CHECK-NEXT: v_mov_b32_e32 v47, v3
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[10:13] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v48
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 10, v[22:23]
+; CHECK-NEXT: v_mov_b32_e32 v3, v45
+; CHECK-NEXT: v_and_b32_e32 v45, 0x7fffffff, v18
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 22, v22
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 9, v[44:45]
+; CHECK-NEXT: v_mov_b32_e32 v45, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v43
+; CHECK-NEXT: v_and_b32_e32 v43, 0x7fffffff, v16
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v44
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 8, v[42:43]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_and_b32_e32 v39, 0x7fffffff, v14
+; CHECK-NEXT: v_mov_b32_e32 v43, v3
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[6:7] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 24, v42
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 7, v[38:39]
+; CHECK-NEXT: v_mov_b32_e32 v3, v41
+; CHECK-NEXT: v_and_b32_e32 v41, 0x7fffffff, v12
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[10:13] offset:48
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v38
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 6, v[40:41]
+; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v10
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 26, v40
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 5, v[32:33]
+; CHECK-NEXT: v_and_b32_e32 v37, 0x7fffffff, v8
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[6:7] offset:32
+; CHECK-NEXT: v_or_b32_e32 v11, v11, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 27, v32
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 4, v[36:37]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_and_b32_e32 v55, 0x7fffffff, v6
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[6:7] offset:32
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 28, v36
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 3, v[54:55]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v4
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[10:13] offset:16
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[6:7]
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v54
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[16:17]
+; CHECK-NEXT: v_mov_b32_e32 v41, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v15
; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[0:1] offset:16
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v12
-; CHECK-NEXT: v_or_b32_e32 v11, v11, v0
-; CHECK-NEXT: flat_store_dwordx4 v[52:53], v[8:11]
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v2
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[0:1] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v16
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 1, v[14:15]
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v0
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[4:7]
; CHECK-NEXT: .LBB0_43: ; %Flow48
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
; CHECK-NEXT: s_andn2_b64 s[48:49], s[48:49], exec
@@ -1121,510 +824,285 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_xor_b64 s[46:47], exec, s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_47
; CHECK-NEXT: ; %bb.46: ; %bb1b
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
-; CHECK-NEXT: v_lshrrev_b32_e32 v33, 15, v31
-; CHECK-NEXT: v_alignbit_b32 v36, v31, v32, 15
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1564 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1568 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1572 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1576 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1580 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1584 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1588 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1592 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1596 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1600 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1604 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1608 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1612 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1616 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1620 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1624 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v32
-; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v59
-; CHECK-NEXT: v_and_b32_e32 v51, 0x7fffffff, v51
-; CHECK-NEXT: v_and_b32_e32 v57, 0x7fffffff, v57
-; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v49
-; CHECK-NEXT: v_and_b32_e32 v43, 0x7fffffff, v43
-; CHECK-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_short v[0:1], v33
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_dword v[0:1], v36
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v42
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
-; CHECK-NEXT: v_or_b32_e32 v41, v4, v0
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[8:9]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a33
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v8
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a32
-; CHECK-NEXT: v_lshrrev_b64 v[42:43], 1, v[42:43]
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v34
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
+; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v30
+; CHECK-NEXT: flat_store_short v[0:1], v2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[6:7]
+; CHECK-NEXT: v_accvgpr_read_b32 v3, a9
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v34
+; CHECK-NEXT: v_alignbit_b32 v2, v30, v3, 15
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a46
+; CHECK-NEXT: flat_store_dword v[0:1], v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v8
+; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v0
+; CHECK-NEXT: v_or_b32_e32 v11, v1, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v3
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a32
+; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v28
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 14, v[2:3]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a22
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v26
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v0
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v34
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 18, v2
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 13, v[12:13]
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[8:9]
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a35
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a34
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a25
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v8
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[6:7]
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a24
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:80
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a20
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v24
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 12, v[4:5]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a18
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v22
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v4
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 11, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[8:9]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a23
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v8
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[58:59]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[2:3] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v58
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[50:51]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a22
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a21
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v50
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[6:7]
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a20
-; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:64
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a16
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v20
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 10, v[4:5]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a14
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v18
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 22, v4
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 9, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v6
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[8:9]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a7
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v8
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[56:57]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a6
-; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[2:3] offset:32
-; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v56
-; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[6:7]
-; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
-; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 23, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a12
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v16
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 8, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 24, v6
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a10
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v14
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[4:5] offset:64
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 7, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_accvgpr_read_b32 v10, a48
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5] offset:32
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a8
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v12
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v6
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], 6, v[4:5]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a6
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v10
+; CHECK-NEXT: v_or_b32_e32 v7, v7, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 26, v4
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 5, v[12:13]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 27, v12
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a62
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v8
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 4, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 28, v6
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a58
+; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v6
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[4:5] offset:32
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 3, v[6:7]
+; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a50
+; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v4
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v6
-; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[48:49]
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[4:5]
; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
-; CHECK-NEXT: flat_store_dwordx2 v[10:11], v[0:1] offset:16
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v48
-; CHECK-NEXT: v_or_b32_e32 v43, v43, v0
-; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[40:43]
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
-; CHECK-NEXT: ; kill: killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v2
+; CHECK-NEXT: flat_store_dwordx2 v[34:35], v[0:1] offset:16
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v4
+; CHECK-NEXT: v_lshrrev_b64 v[12:13], 1, v[8:9]
+; CHECK-NEXT: v_or_b32_e32 v13, v13, v0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[10:13]
+; CHECK-NEXT: ; implicit-def: $agpr57
+; CHECK-NEXT: ; implicit-def: $vgpr41
+; CHECK-NEXT: ; implicit-def: $vgpr43
+; CHECK-NEXT: ; implicit-def: $vgpr45
+; CHECK-NEXT: ; implicit-def: $vgpr62
+; CHECK-NEXT: ; implicit-def: $vgpr27
+; CHECK-NEXT: ; implicit-def: $vgpr58
+; CHECK-NEXT: ; implicit-def: $vgpr59
+; CHECK-NEXT: ; implicit-def: $vgpr47
+; CHECK-NEXT: ; implicit-def: $vgpr57
+; CHECK-NEXT: ; implicit-def: $vgpr60
+; CHECK-NEXT: ; implicit-def: $vgpr61
+; CHECK-NEXT: ; implicit-def: $vgpr8
+; CHECK-NEXT: ; implicit-def: $vgpr9
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $vgpr0
+; CHECK-NEXT: ; kill: killed $vgpr0
+; CHECK-NEXT: ; implicit-def: $agpr15
+; CHECK-NEXT: ; implicit-def: $agpr17
+; CHECK-NEXT: ; implicit-def: $agpr7
+; CHECK-NEXT: ; implicit-def: $agpr19
+; CHECK-NEXT: ; implicit-def: $agpr11
+; CHECK-NEXT: ; implicit-def: $agpr21
+; CHECK-NEXT: ; implicit-def: $agpr5
+; CHECK-NEXT: ; implicit-def: $vgpr3
+; CHECK-NEXT: ; implicit-def: $agpr23
+; CHECK-NEXT: ; implicit-def: $agpr29
+; CHECK-NEXT: ; implicit-def: $agpr31
+; CHECK-NEXT: ; implicit-def: $agpr25
+; CHECK-NEXT: ; implicit-def: $agpr27
+; CHECK-NEXT: ; implicit-def: $agpr34
+; CHECK-NEXT: ; implicit-def: $agpr35
+; CHECK-NEXT: ; implicit-def: $agpr13
+; CHECK-NEXT: ; implicit-def: $agpr33
+; CHECK-NEXT: ; implicit-def: $agpr39
+; CHECK-NEXT: ; implicit-def: $agpr40
+; CHECK-NEXT: ; implicit-def: $agpr37
+; CHECK-NEXT: ; implicit-def: $agpr38
+; CHECK-NEXT: ; implicit-def: $agpr43
+; CHECK-NEXT: ; implicit-def: $agpr45
+; CHECK-NEXT: ; implicit-def: $agpr41
+; CHECK-NEXT: ; implicit-def: $agpr42
+; CHECK-NEXT: ; implicit-def: $agpr51
+; CHECK-NEXT: ; implicit-def: $agpr52
+; CHECK-NEXT: ; implicit-def: $agpr47
+; CHECK-NEXT: ; implicit-def: $agpr49
+; CHECK-NEXT: ; implicit-def: $agpr55
+; CHECK-NEXT: ; implicit-def: $agpr56
+; CHECK-NEXT: ; implicit-def: $agpr53
+; CHECK-NEXT: ; implicit-def: $agpr54
+; CHECK-NEXT: ; implicit-def: $vgpr52
+; CHECK-NEXT: ; implicit-def: $agpr59
+; CHECK-NEXT: ; implicit-def: $vgpr53
+; CHECK-NEXT: ; implicit-def: $agpr60
+; CHECK-NEXT: ; implicit-def: $agpr61
+; CHECK-NEXT: ; implicit-def: $agpr63
+; CHECK-NEXT: ; implicit-def: $vgpr35
; CHECK-NEXT: .LBB0_47: ; %Flow49
; CHECK-NEXT: s_andn2_saveexec_b64 s[46:47], s[46:47]
; CHECK-NEXT: s_cbranch_execz .LBB0_49
; CHECK-NEXT: ; %bb.48: ; %bb1a
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a3, off, s[0:3], s32 offset:1284 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a4, off, s[0:3], s32 offset:1288 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a5, off, s[0:3], s32 offset:1292 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:1296 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a7, off, s[0:3], s32 offset:1300 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a8, off, s[0:3], s32 offset:1304 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a9, off, s[0:3], s32 offset:1308 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a10, off, s[0:3], s32 offset:1312 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a11, off, s[0:3], s32 offset:1316 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a12, off, s[0:3], s32 offset:1320 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a13, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a15, off, s[0:3], s32 offset:1332 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a16, off, s[0:3], s32 offset:1336 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a25, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a27, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:992 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_accvgpr_read_b32 v15, a43
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a45
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a25
+; CHECK-NEXT: v_cndmask_b32_e64 v7, v62, v6, s[6:7]
+; CHECK-NEXT: v_accvgpr_read_b32 v6, a27
+; CHECK-NEXT: v_cndmask_b32_e64 v6, v27, v6, s[6:7]
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a34
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v58, v4, s[8:9]
+; CHECK-NEXT: v_accvgpr_read_b32 v4, a35
+; CHECK-NEXT: v_accvgpr_read_b32 v10, a39
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a29
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a57
+; CHECK-NEXT: v_cndmask_b32_e64 v4, v59, v4, s[8:9]
+; CHECK-NEXT: v_cndmask_b32_e64 v11, v60, v10, s[16:17]
+; CHECK-NEXT: v_accvgpr_read_b32 v10, a40
+; CHECK-NEXT: v_cndmask_b32_e64 v13, v47, v12, s[12:13]
+; CHECK-NEXT: v_accvgpr_read_b32 v12, a33
+; CHECK-NEXT: v_cndmask_b32_e64 v1, v43, v0, s[10:11]
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a31
+; CHECK-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a23
+; CHECK-NEXT: v_cndmask_b32_e64 v10, v61, v10, s[16:17]
+; CHECK-NEXT: v_cndmask_b32_e64 v12, v57, v12, s[12:13]
+; CHECK-NEXT: v_cndmask_b32_e64 v0, v45, v0, s[10:11]
+; CHECK-NEXT: v_cndmask_b32_e64 v2, v41, v2, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v15, v14, v15, s[18:19]
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v14, v14, v16, s[18:19]
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a37
+; CHECK-NEXT: v_cndmask_b32_e64 v17, v8, v16, s[14:15]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a38
+; CHECK-NEXT: v_cndmask_b32_e64 v16, v9, v8, s[14:15]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a51
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v19, v8, v9, s[44:45]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a52
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a34
-; CHECK-NEXT: v_accvgpr_read_b32 v36, a38
-; CHECK-NEXT: v_accvgpr_read_b32 v37, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v48, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v59, a61
-; CHECK-NEXT: v_accvgpr_read_b32 v49, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v53, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v41, a43
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v38, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a35
-; CHECK-NEXT: v_accvgpr_read_b32 v39, a41
-; CHECK-NEXT: v_accvgpr_read_b32 v52, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v40, a42
-; CHECK-NEXT: v_accvgpr_read_b32 v51, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v50, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v55, a57
-; CHECK-NEXT: v_accvgpr_read_b32 v43, a45
-; CHECK-NEXT: v_accvgpr_read_b32 v54, a56
-; CHECK-NEXT: v_accvgpr_read_b32 v42, a44
-; CHECK-NEXT: v_accvgpr_read_b32 v35, a37
-; CHECK-NEXT: v_accvgpr_read_b32 v44, a46
-; CHECK-NEXT: v_accvgpr_read_b32 v45, a47
-; CHECK-NEXT: v_accvgpr_read_b32 v46, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v47, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v56, a58
-; CHECK-NEXT: v_accvgpr_read_b32 v57, a59
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a60
-; CHECK-NEXT: v_accvgpr_read_b32 v60, a62
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v61, a63
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a30
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a24
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a60
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a56
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a25
-; CHECK-NEXT: v_cndmask_b32_e64 v36, v28, v32, s[44:45]
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a29
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a56
-; CHECK-NEXT: v_cndmask_b32_e64 v37, v59, v29, s[44:45]
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a25
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a57
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
-; CHECK-NEXT: v_cndmask_b32_e64 v48, v24, v28, s[42:43]
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a20
-; CHECK-NEXT: v_cndmask_b32_e64 v49, v25, v29, s[42:43]
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v13, a13
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a17
-; CHECK-NEXT: v_cndmask_b32_e64 v53, v21, v29, s[28:29]
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a9
-; CHECK-NEXT: v_cndmask_b32_e64 v41, v17, v29, s[24:25]
-; CHECK-NEXT: v_accvgpr_read_b32 v13, a45
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a13
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a8
+; CHECK-NEXT: v_cndmask_b32_e64 v18, v8, v9, s[44:45]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v9, a41
-; CHECK-NEXT: v_cndmask_b32_e64 v17, v13, v29, s[20:21]
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a5
-; CHECK-NEXT: v_cndmask_b32_e64 v21, v9, v29, s[16:17]
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a37
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
-; CHECK-NEXT: v_cndmask_b32_e64 v25, v5, v29, s[12:13]
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_cndmask_b32_e64 v29, v1, v29, s[4:5]
-; CHECK-NEXT: v_cndmask_b32_e64 v28, v0, v28, s[4:5]
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
-; CHECK-NEXT: v_cndmask_b32_e64 v38, v30, v34, s[40:41]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a33
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a59
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a31
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a63
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v12, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a18
-; CHECK-NEXT: v_cndmask_b32_e64 v39, v31, v33, s[40:41]
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a59
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
-; CHECK-NEXT: v_cndmask_b32_e64 v52, v20, v24, s[28:29]
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_cndmask_b32_e64 v51, v27, v33, s[26:27]
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
-; CHECK-NEXT: v_cndmask_b32_e64 v40, v16, v20, s[24:25]
-; CHECK-NEXT: v_accvgpr_read_b32 v12, a44
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a4
-; CHECK-NEXT: v_cndmask_b32_e64 v16, v12, v30, s[20:21]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v15, a15
-; CHECK-NEXT: v_cndmask_b32_e64 v50, v26, v34, s[26:27]
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
-; CHECK-NEXT: v_cndmask_b32_e64 v55, v23, v33, s[22:23]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v11, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v15, a47
-; CHECK-NEXT: v_cndmask_b32_e64 v43, v19, v33, s[18:19]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a15
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a15
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a2
-; CHECK-NEXT: v_cndmask_b32_e64 v20, v8, v30, s[16:17]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v14, a14
-; CHECK-NEXT: v_cndmask_b32_e64 v54, v22, v34, s[22:23]
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v11, a43
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a6
-; CHECK-NEXT: v_cndmask_b32_e64 v19, v15, v33, s[14:15]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v3, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v10, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v14, a46
-; CHECK-NEXT: v_cndmask_b32_e64 v42, v18, v34, s[18:19]
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a4
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v11, v33, s[10:11]
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a7
-; CHECK-NEXT: v_cndmask_b32_e64 v24, v4, v30, s[12:13]
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a34
-; CHECK-NEXT: v_accvgpr_read_b32 v3, a35
-; CHECK-NEXT: v_accvgpr_read_b32 v10, a42
-; CHECK-NEXT: v_cndmask_b32_e64 v18, v14, v34, s[14:15]
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
-; CHECK-NEXT: v_cndmask_b32_e64 v27, v7, v33, s[6:7]
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a38
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v10, v34, s[10:11]
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a6
-; CHECK-NEXT: v_cndmask_b32_e64 v31, v3, v33, s[8:9]
-; CHECK-NEXT: v_cndmask_b32_e64 v30, v2, v30, s[8:9]
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a18
-; CHECK-NEXT: v_cndmask_b32_e64 v26, v6, v34, s[6:7]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:16
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:32
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a14
-; CHECK-NEXT: v_add_co_u32_e32 v20, vcc, 48, v0
-; CHECK-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v1, vcc
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a10
-; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x70, v0
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a6
-; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a2
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[16:19]
-; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[40:43] offset:64
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[52:55] offset:32
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[48:51] offset:48
-; CHECK-NEXT: flat_store_dwordx4 v[4:5], v[36:39]
+; CHECK-NEXT: v_cndmask_b32_e64 v21, v8, v9, s[40:41]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a42
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v20, v8, v9, s[40:41]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a55
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v23, v8, v9, s[42:43]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a56
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v22, v8, v9, s[42:43]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a47
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v25, v8, v9, s[22:23]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a49
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v24, v8, v9, s[22:23]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a17
+; CHECK-NEXT: v_cndmask_b32_e64 v27, v8, v52, s[26:27]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a59
+; CHECK-NEXT: v_cndmask_b32_e64 v26, v8, v9, s[26:27]
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a53
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v29, v8, v9, s[20:21]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a54
+; CHECK-NEXT: v_cndmask_b32_e64 v28, v8, v9, s[20:21]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a19
+; CHECK-NEXT: v_cndmask_b32_e64 v33, v8, v53, s[24:25]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a60
+; CHECK-NEXT: v_cndmask_b32_e64 v32, v8, v9, s[24:25]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a21
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a61
+; CHECK-NEXT: v_cndmask_b32_e64 v31, v8, v9, s[28:29]
+; CHECK-NEXT: v_accvgpr_read_b32 v8, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a63
+; CHECK-NEXT: v_cndmask_b32_e64 v30, v8, v9, s[28:29]
+; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, 48, v34
+; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v35, vcc
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[18:21]
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[14:17] offset:64
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[10:13] offset:32
+; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[4:7] offset:48
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[30:33]
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[26:29] offset:16
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[22:25] offset:32
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x70, v34
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v35, vcc
+; CHECK-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; CHECK-NEXT: .LBB0_49: ; %UnifiedReturnBlock
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
@@ -1691,7 +1169,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_readlane_b32 s35, v63, 1
; CHECK-NEXT: v_readlane_b32 s34, v63, 0
; CHECK-NEXT: s_or_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:1632 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index fd713082c1f97..97b62ccd9f0ab 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -391,7 +391,6 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[4:5]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT: v_mov_b32_e32 v13, v12
; GCN-IR-NEXT: v_cndmask_b32_e64 v7, v1, 0, s[4:5]
; GCN-IR-NEXT: v_cndmask_b32_e64 v6, v0, 0, s[4:5]
; GCN-IR-NEXT: s_and_b64 s[4:5], s[6:7], vcc
@@ -409,12 +408,12 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB1_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v14, vcc, -1, v2
-; GCN-IR-NEXT: v_addc_u32_e32 v15, vcc, -1, v3, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v13, vcc, -1, v2
+; GCN-IR-NEXT: v_addc_u32_e32 v14, vcc, -1, v3, vcc
; GCN-IR-NEXT: v_not_b32_e32 v6, v10
-; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, v6, v11
+; GCN-IR-NEXT: v_add_i32_e32 v15, vcc, v6, v11
; GCN-IR-NEXT: v_lshr_b64 v[8:9], v[0:1], v8
-; GCN-IR-NEXT: v_addc_u32_e64 v17, s[8:9], -1, 0, vcc
+; GCN-IR-NEXT: v_addc_u32_e64 v16, s[8:9], -1, 0, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v10, 0
; GCN-IR-NEXT: v_mov_b32_e32 v11, 0
@@ -425,8 +424,8 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_lshrrev_b32_e32 v6, 31, v5
; GCN-IR-NEXT: v_or_b32_e32 v8, v8, v6
; GCN-IR-NEXT: v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v14, v8
-; GCN-IR-NEXT: v_subb_u32_e32 v6, vcc, v15, v9, vcc
+; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v13, v8
+; GCN-IR-NEXT: v_subb_u32_e32 v6, vcc, v14, v9, vcc
; GCN-IR-NEXT: v_or_b32_e32 v4, v10, v4
; GCN-IR-NEXT: v_ashrrev_i32_e32 v10, 31, v6
; GCN-IR-NEXT: v_or_b32_e32 v5, v11, v5
@@ -435,8 +434,8 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_and_b32_e32 v10, v10, v2
; GCN-IR-NEXT: v_sub_i32_e32 v8, vcc, v8, v10
; GCN-IR-NEXT: v_subb_u32_e32 v9, vcc, v9, v11, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v16, vcc, 1, v16
-; GCN-IR-NEXT: v_addc_u32_e32 v17, vcc, 0, v17, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v15, vcc, 1, v15
+; GCN-IR-NEXT: v_addc_u32_e32 v16, vcc, 0, v16, vcc
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v11, v7
; GCN-IR-NEXT: v_mov_b32_e32 v10, v6
@@ -460,9 +459,9 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v12
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v13
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v12
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v12
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v13, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v12, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = srem i64 %x, %y
ret i64 %result
@@ -1020,20 +1019,20 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IR-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN-IR-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IR-NEXT: s_ashr_i32 s7, s3, 31
-; GCN-IR-NEXT: s_mov_b32 s6, s3
-; GCN-IR-NEXT: s_ashr_i32 s3, s5, 31
-; GCN-IR-NEXT: s_mov_b32 s2, s5
-; GCN-IR-NEXT: s_mov_b32 s4, s7
-; GCN-IR-NEXT: s_mov_b32 s5, s7
-; GCN-IR-NEXT: s_xor_b64 s[8:9], s[6:7], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s6, s8, s7
-; GCN-IR-NEXT: s_subb_u32 s7, s9, s7
-; GCN-IR-NEXT: s_mov_b32 s8, s3
-; GCN-IR-NEXT: s_mov_b32 s9, s3
-; GCN-IR-NEXT: s_xor_b64 s[8:9], s[2:3], s[8:9]
-; GCN-IR-NEXT: s_sub_u32 s8, s8, s3
-; GCN-IR-NEXT: s_subb_u32 s9, s9, s3
+; GCN-IR-NEXT: s_ashr_i32 s4, s3, 31
+; GCN-IR-NEXT: s_mov_b32 s2, s3
+; GCN-IR-NEXT: s_ashr_i32 s9, s5, 31
+; GCN-IR-NEXT: s_mov_b32 s8, s5
+; GCN-IR-NEXT: s_mov_b32 s3, s4
+; GCN-IR-NEXT: s_mov_b32 s5, s4
+; GCN-IR-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GCN-IR-NEXT: s_sub_u32 s6, s2, s4
+; GCN-IR-NEXT: s_subb_u32 s7, s3, s4
+; GCN-IR-NEXT: s_mov_b32 s2, s9
+; GCN-IR-NEXT: s_mov_b32 s3, s9
+; GCN-IR-NEXT: s_xor_b64 s[2:3], s[8:9], s[2:3]
+; GCN-IR-NEXT: s_sub_u32 s8, s2, s9
+; GCN-IR-NEXT: s_subb_u32 s9, s3, s9
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[2:3], s[8:9], 0
; GCN-IR-NEXT: v_cmp_eq_u64_e64 s[10:11], s[6:7], 0
; GCN-IR-NEXT: s_flbit_i32_b64 s12, s[8:9]
@@ -1109,12 +1108,12 @@ define amdgpu_kernel void @s_test_srem32_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_sub_u32 s6, s6, s8
; GCN-IR-NEXT: s_subb_u32 s7, s7, s11
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s4, s6, s4
-; GCN-IR-NEXT: s_subb_u32 s5, s7, s5
+; GCN-IR-NEXT: s_sub_u32 s6, s6, s4
+; GCN-IR-NEXT: s_subb_u32 s7, s7, s4
; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
; GCN-IR-NEXT: s_mov_b32 s2, -1
-; GCN-IR-NEXT: v_mov_b32_e32 v0, s4
-; GCN-IR-NEXT: v_mov_b32_e32 v1, s5
+; GCN-IR-NEXT: v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT: v_mov_b32_e32 v1, s7
; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 32
@@ -1375,12 +1374,12 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 %
; GCN-IR-NEXT: s_sub_u32 s6, s6, s8
; GCN-IR-NEXT: s_subb_u32 s7, s7, s11
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
-; GCN-IR-NEXT: s_sub_u32 s4, s6, s4
-; GCN-IR-NEXT: s_subb_u32 s5, s7, s5
+; GCN-IR-NEXT: s_sub_u32 s6, s6, s4
+; GCN-IR-NEXT: s_subb_u32 s7, s7, s4
; GCN-IR-NEXT: s_mov_b32 s3, 0xf000
; GCN-IR-NEXT: s_mov_b32 s2, -1
-; GCN-IR-NEXT: v_mov_b32_e32 v0, s4
-; GCN-IR-NEXT: v_mov_b32_e32 v1, s5
+; GCN-IR-NEXT: v_mov_b32_e32 v0, s6
+; GCN-IR-NEXT: v_mov_b32_e32 v1, s7
; GCN-IR-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GCN-IR-NEXT: s_endpgm
%1 = ashr i64 %x, 31
@@ -2119,7 +2118,6 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3]
-; GCN-IR-NEXT: v_mov_b32_e32 v11, v10
; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
@@ -2140,9 +2138,9 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB13_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
-; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc
+; GCN-IR-NEXT: v_addc_u32_e64 v12, s[8:9], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
@@ -2162,8 +2160,8 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8
; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v12
-; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, 1, v11
+; GCN-IR-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v9, v5
@@ -2183,9 +2181,9 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v11
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v10
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%result = srem i64 %x, 32768
ret i64 %result
@@ -2480,7 +2478,6 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; GCN-IR-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN-IR-NEXT: v_cmp_lt_u64_e64 s[4:5], 63, v[2:3]
-; GCN-IR-NEXT: v_mov_b32_e32 v11, v10
; GCN-IR-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-IR-NEXT: v_cmp_ne_u64_e32 vcc, 63, v[2:3]
; GCN-IR-NEXT: s_xor_b64 s[6:7], s[4:5], -1
@@ -2501,9 +2498,9 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
; GCN-IR-NEXT: s_cbranch_execz .LBB18_5
; GCN-IR-NEXT: ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 0xffffffcf, v8
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, 0xffffffcf, v8
; GCN-IR-NEXT: v_lshr_b64 v[6:7], v[0:1], v6
-; GCN-IR-NEXT: v_addc_u32_e64 v13, s[8:9], 0, -1, vcc
+; GCN-IR-NEXT: v_addc_u32_e64 v12, s[8:9], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[8:9], 0
; GCN-IR-NEXT: v_mov_b32_e32 v8, 0
; GCN-IR-NEXT: v_mov_b32_e32 v9, 0
@@ -2523,8 +2520,8 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_and_b32_e32 v8, 0x8000, v8
; GCN-IR-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
; GCN-IR-NEXT: v_subbrev_u32_e32 v7, vcc, 0, v7, vcc
-; GCN-IR-NEXT: v_add_i32_e32 v12, vcc, 1, v12
-; GCN-IR-NEXT: v_addc_u32_e32 v13, vcc, 0, v13, vcc
+; GCN-IR-NEXT: v_add_i32_e32 v11, vcc, 1, v11
+; GCN-IR-NEXT: v_addc_u32_e32 v12, vcc, 0, v12, vcc
; GCN-IR-NEXT: v_or_b32_e32 v3, v9, v3
; GCN-IR-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GCN-IR-NEXT: v_mov_b32_e32 v9, v5
@@ -2544,9 +2541,9 @@ define i64 @v_test_srem24_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
; GCN-IR-NEXT: v_xor_b32_e32 v0, v0, v10
-; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v11
+; GCN-IR-NEXT: v_xor_b32_e32 v1, v1, v10
; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
-; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v11, vcc
+; GCN-IR-NEXT: v_subb_u32_e32 v1, vcc, v1, v10, vcc
; GCN-IR-NEXT: s_setpc_b64 s[30:31]
%x.shr = ashr i64 %x, 40
%result = srem i64 %x.shr, 32768
diff --git a/llvm/test/CodeGen/AMDGPU/swdev373493.ll b/llvm/test/CodeGen/AMDGPU/swdev373493.ll
index 914dcccd6969f..baf903e7cb366 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev373493.ll
+++ b/llvm/test/CodeGen/AMDGPU/swdev373493.ll
@@ -7,21 +7,17 @@ define hidden fastcc void @bar(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %a
; CHECK-LABEL: bar:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v15, v12
-; CHECK-NEXT: v_mov_b32_e32 v14, v11
; CHECK-NEXT: v_mov_b32_e32 v13, v10
-; CHECK-NEXT: v_mov_b32_e32 v12, v9
-; CHECK-NEXT: v_mov_b32_e32 v11, v8
+; CHECK-NEXT: v_mov_b32_e32 v14, v9
+; CHECK-NEXT: v_mov_b32_e32 v15, v8
; CHECK-NEXT: v_mov_b32_e32 v10, v7
; CHECK-NEXT: v_mov_b32_e32 v9, v6
; CHECK-NEXT: v_mov_b32_e32 v8, v5
-; CHECK-NEXT: v_mov_b32_e32 v7, v4
-; CHECK-NEXT: v_mov_b32_e32 v6, v3
; CHECK-NEXT: s_branch .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %LeafBlock
; CHECK-NEXT: s_cbranch_scc1 .LBB0_5
; CHECK-NEXT: ; %bb.2: ; %bb7
-; CHECK-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; CHECK-NEXT: flat_load_dwordx2 v[16:17], v[0:1]
; CHECK-NEXT: s_getpc_b64 s[18:19]
; CHECK-NEXT: s_add_u32 s18, s18, global at rel32@lo+1948
; CHECK-NEXT: s_addc_u32 s19, s19, global at rel32@hi+1956
@@ -29,27 +25,34 @@ define hidden fastcc void @bar(i32 %arg, ptr %arg1, ptr %arg2, ptr %arg3, ptr %a
; CHECK-NEXT: s_add_u32 s16, s16, eggs at rel32@lo+4
; CHECK-NEXT: s_addc_u32 s17, s17, eggs at rel32@hi+12
; CHECK-NEXT: v_mov_b32_e32 v5, 0
+; CHECK-NEXT: v_mov_b32_e32 v6, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, v4
+; CHECK-NEXT: v_mov_b32_e32 v11, v15
+; CHECK-NEXT: v_mov_b32_e32 v12, v14
; CHECK-NEXT: v_mov_b32_e32 v0, s18
; CHECK-NEXT: v_mov_b32_e32 v1, s19
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v2, v16
+; CHECK-NEXT: v_mov_b32_e32 v3, v17
; CHECK-NEXT: s_setpc_b64 s[16:17]
; CHECK-NEXT: .LBB0_3: ; %LeafBlock1
; CHECK-NEXT: s_cbranch_scc0 .LBB0_5
; CHECK-NEXT: ; %bb.4: ; %bb8
-; CHECK-NEXT: v_mov_b32_e32 v0, v1
; CHECK-NEXT: s_getpc_b64 s[16:17]
; CHECK-NEXT: s_add_u32 s16, s16, quux at rel32@lo+4
; CHECK-NEXT: s_addc_u32 s17, s17, quux at rel32@hi+12
+; CHECK-NEXT: v_mov_b32_e32 v0, v1
; CHECK-NEXT: v_mov_b32_e32 v1, v2
-; CHECK-NEXT: v_mov_b32_e32 v2, v6
-; CHECK-NEXT: v_mov_b32_e32 v3, v7
+; CHECK-NEXT: v_mov_b32_e32 v2, v3
+; CHECK-NEXT: v_mov_b32_e32 v3, v4
; CHECK-NEXT: v_mov_b32_e32 v4, v8
; CHECK-NEXT: v_mov_b32_e32 v5, v9
; CHECK-NEXT: v_mov_b32_e32 v6, v10
-; CHECK-NEXT: v_mov_b32_e32 v7, v11
-; CHECK-NEXT: v_mov_b32_e32 v8, v12
+; CHECK-NEXT: v_mov_b32_e32 v7, v15
+; CHECK-NEXT: v_mov_b32_e32 v8, v14
; CHECK-NEXT: v_mov_b32_e32 v9, v13
-; CHECK-NEXT: v_mov_b32_e32 v10, v14
-; CHECK-NEXT: v_mov_b32_e32 v11, v15
+; CHECK-NEXT: v_mov_b32_e32 v10, v11
+; CHECK-NEXT: v_mov_b32_e32 v11, v12
; CHECK-NEXT: s_setpc_b64 s[16:17]
; CHECK-NEXT: .LBB0_5: ; %bb9
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index 61ad66d2b3b33..e9b597be5f141 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -6,35 +6,34 @@ define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 2, v0
+; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 2, v4
; GFX942-NEXT: v_mov_b32_e32 v2, 8
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v3, v1, s[0:1]
; GFX942-NEXT: s_mov_b32 s4, 0xff0000
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: v_lshrrev_b32_sdwa v5, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_or_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX942-NEXT: v_and_or_b32 v3, v3, s4, v4
+; GFX942-NEXT: v_or_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX942-NEXT: v_and_or_b32 v3, v3, s4, v5
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB0_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX942-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX942-NEXT: global_load_dword v1, v1, s[2:3]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX942-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX942-NEXT: v_and_or_b32 v3, v0, s4, v2
+; GFX942-NEXT: v_and_or_b32 v3, v1, s4, v2
; GFX942-NEXT: .LBB0_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: global_store_byte_d16_hi v1, v3, s[6:7] offset:2
-; GFX942-NEXT: global_store_short v1, v3, s[6:7]
+; GFX942-NEXT: global_store_byte_d16_hi v0, v3, s[6:7] offset:2
+; GFX942-NEXT: global_store_short v0, v3, s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -58,21 +57,20 @@ define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 2, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 2, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v2, v1, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v3
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB1_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX942-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX942-NEXT: global_load_dword v2, v1, s[2:3]
; GFX942-NEXT: .LBB1_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dword v1, v2, s[6:7]
+; GFX942-NEXT: global_store_dword v0, v2, s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -96,25 +94,24 @@ define amdgpu_kernel void @v5i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v4
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v2
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[0:1]
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB2_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[2:3]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX942-NEXT: .LBB2_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: global_store_byte v3, v1, s[6:7] offset:4
-; GFX942-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX942-NEXT: global_store_byte v2, v1, s[6:7] offset:4
+; GFX942-NEXT: global_store_dword v2, v0, s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -138,21 +135,20 @@ define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB3_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX942-NEXT: global_load_dwordx2 v[2:3], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[2:3]
; GFX942-NEXT: .LBB3_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[6:7]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -176,21 +172,20 @@ define amdgpu_kernel void @v16i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 4, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v6, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 4, v6
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v6
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB4_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3]
; GFX942-NEXT: .LBB4_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -214,25 +209,24 @@ define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 5, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v10, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 5, v10
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:16
; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v10
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB5_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 5, v0
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3]
; GFX942-NEXT: .LBB5_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:16
+; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] offset:16
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -256,9 +250,9 @@ define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v62, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v62
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:240
; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:224
@@ -276,53 +270,52 @@ define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:32
; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:16
; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v62
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB6_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX942-NEXT: global_load_dwordx4 v[30:33], v0, s[2:3] offset:240
-; GFX942-NEXT: global_load_dwordx4 v[26:29], v0, s[2:3] offset:224
-; GFX942-NEXT: global_load_dwordx4 v[22:25], v0, s[2:3] offset:208
-; GFX942-NEXT: global_load_dwordx4 v[18:21], v0, s[2:3] offset:192
-; GFX942-NEXT: global_load_dwordx4 v[14:17], v0, s[2:3] offset:176
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v0, s[2:3] offset:160
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:144
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:128
-; GFX942-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3] offset:112
-; GFX942-NEXT: global_load_dwordx4 v[58:61], v0, s[2:3] offset:96
-; GFX942-NEXT: global_load_dwordx4 v[54:57], v0, s[2:3] offset:80
-; GFX942-NEXT: global_load_dwordx4 v[50:53], v0, s[2:3] offset:64
-; GFX942-NEXT: global_load_dwordx4 v[46:49], v0, s[2:3] offset:48
-; GFX942-NEXT: global_load_dwordx4 v[42:45], v0, s[2:3] offset:32
-; GFX942-NEXT: global_load_dwordx4 v[38:41], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[34:37], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[2:3] offset:240
+; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[2:3] offset:224
+; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[2:3] offset:208
+; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[2:3] offset:192
+; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[2:3] offset:176
+; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[2:3] offset:160
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[2:3] offset:144
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[2:3] offset:128
+; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[2:3] offset:112
+; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[2:3] offset:96
+; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[2:3] offset:80
+; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[2:3] offset:64
+; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[2:3] offset:48
+; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[2:3] offset:32
+; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[2:3]
; GFX942-NEXT: .LBB6_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] offset:112
+; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] offset:112
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[58:61], s[6:7] offset:96
+; GFX942-NEXT: global_store_dwordx4 v0, v[58:61], s[6:7] offset:96
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[54:57], s[6:7] offset:80
+; GFX942-NEXT: global_store_dwordx4 v0, v[54:57], s[6:7] offset:80
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[50:53], s[6:7] offset:64
+; GFX942-NEXT: global_store_dwordx4 v0, v[50:53], s[6:7] offset:64
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[46:49], s[6:7] offset:48
+; GFX942-NEXT: global_store_dwordx4 v0, v[46:49], s[6:7] offset:48
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[42:45], s[6:7] offset:32
+; GFX942-NEXT: global_store_dwordx4 v0, v[42:45], s[6:7] offset:32
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[38:41], s[6:7] offset:16
+; GFX942-NEXT: global_store_dwordx4 v0, v[38:41], s[6:7] offset:16
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[34:37], s[6:7]
-; GFX942-NEXT: global_store_dwordx4 v1, v[30:33], s[6:7] offset:240
-; GFX942-NEXT: global_store_dwordx4 v1, v[26:29], s[6:7] offset:224
-; GFX942-NEXT: global_store_dwordx4 v1, v[22:25], s[6:7] offset:208
-; GFX942-NEXT: global_store_dwordx4 v1, v[18:21], s[6:7] offset:192
-; GFX942-NEXT: global_store_dwordx4 v1, v[14:17], s[6:7] offset:176
-; GFX942-NEXT: global_store_dwordx4 v1, v[10:13], s[6:7] offset:160
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:144
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:128
+; GFX942-NEXT: global_store_dwordx4 v0, v[34:37], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v0, v[30:33], s[6:7] offset:240
+; GFX942-NEXT: global_store_dwordx4 v0, v[26:29], s[6:7] offset:224
+; GFX942-NEXT: global_store_dwordx4 v0, v[22:25], s[6:7] offset:208
+; GFX942-NEXT: global_store_dwordx4 v0, v[18:21], s[6:7] offset:192
+; GFX942-NEXT: global_store_dwordx4 v0, v[14:17], s[6:7] offset:176
+; GFX942-NEXT: global_store_dwordx4 v0, v[10:13], s[6:7] offset:160
+; GFX942-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] offset:144
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7] offset:128
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -407,7 +400,6 @@ define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB8_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[10:11]
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0
; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
@@ -465,7 +457,6 @@ define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspa
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB9_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX942-NEXT: global_load_dwordx2 v[4:5], v1, s[10:11]
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0
; GFX942-NEXT: s_waitcnt vmcnt(1)
@@ -516,80 +507,79 @@ define amdgpu_kernel void @v8i8_phi_const(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-LABEL: v8i8_phi_const:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v4
-; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v4
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
+; GFX942-NEXT: v_and_b32_e32 v16, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v16
+; GFX942-NEXT: v_cmp_lt_u32_e64 s[0:1], 14, v16
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v16
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[8:9]
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[8:9]
; GFX942-NEXT: ; implicit-def: $vgpr2
+; GFX942-NEXT: ; implicit-def: $vgpr10
; GFX942-NEXT: ; implicit-def: $vgpr11
; GFX942-NEXT: ; implicit-def: $vgpr12
; GFX942-NEXT: ; implicit-def: $vgpr13
; GFX942-NEXT: ; implicit-def: $vgpr14
; GFX942-NEXT: ; implicit-def: $vgpr15
-; GFX942-NEXT: ; implicit-def: $vgpr16
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX942-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX942-NEXT: v_lshrrev_b32_e32 v7, 8, v1
-; GFX942-NEXT: v_lshrrev_b32_e32 v8, 24, v0
-; GFX942-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX942-NEXT: v_lshrrev_b32_e32 v10, 8, v0
+; GFX942-NEXT: v_lshrrev_b32_e32 v4, 24, v1
+; GFX942-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX942-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX942-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX942-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX942-NEXT: v_lshrrev_b32_e32 v9, 8, v0
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB10_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v4
-; GFX942-NEXT: global_load_dwordx2 v[2:3], v0, s[10:11]
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v4
+; GFX942-NEXT: global_load_dwordx2 v[2:3], v3, s[10:11]
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v16
; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX942-NEXT: v_mov_b32_e32 v5, 8
-; GFX942-NEXT: v_mov_b32_e32 v6, 7
-; GFX942-NEXT: v_mov_b32_e32 v7, 6
+; GFX942-NEXT: v_mov_b32_e32 v4, 8
+; GFX942-NEXT: v_mov_b32_e32 v5, 7
+; GFX942-NEXT: v_mov_b32_e32 v6, 6
; GFX942-NEXT: v_mov_b32_e32 v1, 5
-; GFX942-NEXT: v_mov_b32_e32 v8, 4
-; GFX942-NEXT: v_mov_b32_e32 v9, 3
-; GFX942-NEXT: v_mov_b32_e32 v10, 2
+; GFX942-NEXT: v_mov_b32_e32 v7, 4
+; GFX942-NEXT: v_mov_b32_e32 v8, 3
+; GFX942-NEXT: v_mov_b32_e32 v9, 2
; GFX942-NEXT: v_mov_b32_e32 v0, 1
; GFX942-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v16, 24, v3
-; GFX942-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX942-NEXT: v_lshrrev_b32_e32 v14, 8, v3
-; GFX942-NEXT: v_lshrrev_b32_e32 v13, 24, v2
-; GFX942-NEXT: v_lshrrev_b32_e32 v12, 16, v2
-; GFX942-NEXT: v_lshrrev_b32_e32 v11, 8, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX942-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX942-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX942-NEXT: v_lshrrev_b32_e32 v12, 24, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v11, 16, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v10, 8, v2
; GFX942-NEXT: .LBB10_2: ; %Flow
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB10_4
; GFX942-NEXT: ; %bb.3: ; %bb.2
; GFX942-NEXT: s_mov_b32 s0, 0xc0c0004
-; GFX942-NEXT: v_perm_b32 v2, v0, v10, s0
-; GFX942-NEXT: v_perm_b32 v3, v9, v8, s0
+; GFX942-NEXT: v_perm_b32 v2, v0, v9, s0
+; GFX942-NEXT: v_perm_b32 v3, v8, v7, s0
; GFX942-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX942-NEXT: v_perm_b32 v3, v1, v7, s0
-; GFX942-NEXT: v_perm_b32 v11, v6, v5, s0
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_perm_b32 v3, v1, v6, s0
+; GFX942-NEXT: v_perm_b32 v11, v5, v4, s0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
; GFX942-NEXT: v_lshl_or_b32 v3, v11, 16, v3
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[12:13]
+; GFX942-NEXT: global_store_dwordx2 v10, v[2:3], s[12:13]
; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v10
-; GFX942-NEXT: v_mov_b32_e32 v12, v9
-; GFX942-NEXT: v_mov_b32_e32 v13, v8
+; GFX942-NEXT: v_mov_b32_e32 v10, v9
+; GFX942-NEXT: v_mov_b32_e32 v11, v8
+; GFX942-NEXT: v_mov_b32_e32 v12, v7
; GFX942-NEXT: v_mov_b32_e32 v3, v1
-; GFX942-NEXT: v_mov_b32_e32 v14, v7
-; GFX942-NEXT: v_mov_b32_e32 v15, v6
-; GFX942-NEXT: v_mov_b32_e32 v16, v5
+; GFX942-NEXT: v_mov_b32_e32 v13, v6
+; GFX942-NEXT: v_mov_b32_e32 v14, v5
+; GFX942-NEXT: v_mov_b32_e32 v15, v4
; GFX942-NEXT: .LBB10_4: ; %bb.3
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_mov_b32 s0, 0xc0c0004
-; GFX942-NEXT: v_perm_b32 v0, v2, v11, s0
-; GFX942-NEXT: v_perm_b32 v1, v12, v13, s0
+; GFX942-NEXT: v_perm_b32 v0, v2, v10, s0
+; GFX942-NEXT: v_perm_b32 v1, v11, v12, s0
; GFX942-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX942-NEXT: v_perm_b32 v1, v3, v14, s0
-; GFX942-NEXT: v_perm_b32 v2, v15, v16, s0
+; GFX942-NEXT: v_perm_b32 v1, v3, v13, s0
+; GFX942-NEXT: v_perm_b32 v2, v14, v15, s0
; GFX942-NEXT: v_mov_b32_e32 v4, 0
; GFX942-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[14:15]
@@ -621,31 +611,30 @@ define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspac
; GFX942-LABEL: v8i8_multi_block:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v1
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[4:5], v1, s[8:9]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: global_load_dwordx2 v[6:7], v2, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB11_4
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX942-NEXT: global_load_dwordx2 v[2:3], v2, s[10:11]
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0
+; GFX942-NEXT: global_load_dwordx2 v[4:5], v2, s[10:11]
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB11_3
; GFX942-NEXT: ; %bb.2: ; %bb.2
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_store_dwordx2 v0, v[4:5], s[12:13]
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: global_store_dwordx2 v1, v[6:7], s[12:13]
; GFX942-NEXT: .LBB11_3: ; %Flow
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: .LBB11_4: ; %bb.3
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[14:15]
+; GFX942-NEXT: global_store_dwordx2 v0, v[4:5], s[14:15]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -866,17 +855,16 @@ define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942-LABEL: v8i8_mfma_i8:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v4
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[8:9]
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB14_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX942-NEXT: global_load_dwordx2 v[2:3], v0, s[10:11]
+; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[10:11]
; GFX942-NEXT: .LBB14_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[14:15], 0x0
@@ -887,7 +875,7 @@ define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mfma_i32_16x16x32_i8 v[2:5], v[2:3], v[2:3], v[4:7] cbsz:1 abid:2 blgp:3
; GFX942-NEXT: s_nop 6
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[12:13]
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[12:13]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -915,16 +903,15 @@ define amdgpu_kernel void @v8i8_mfma_half(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-LABEL: v8i8_mfma_half:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx8 s[36:43], s[4:5], 0x24
-; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX942-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v1
; GFX942-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[34:35], v1, s[36:37]
+; GFX942-NEXT: global_load_dwordx2 v[34:35], v0, s[36:37]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB15_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX942-NEXT: global_load_dwordx2 v[34:35], v0, s[38:39]
; GFX942-NEXT: .LBB15_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1003,23 +990,21 @@ define amdgpu_kernel void @v8i8_intrinsic(ptr addrspace(1) %src1, ptr addrspace(
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v4
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]
-; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v2
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[0:1]
+; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v4
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB16_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx2 v[0:1], v3, s[2:3]
; GFX942-NEXT: .LBB16_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[0:1], v[0:1]
-; GFX942-NEXT: global_store_dwordx2 v3, v[0:1], s[6:7]
+; GFX942-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index b453f925b94e8..a4fda1558a08b 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -242,7 +242,6 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
; AVX1-NEXT: cmpl %r9d, %edx
; AVX1-NEXT: jne .LBB0_6
; AVX1-NEXT: .LBB0_9: # %for.cond.cleanup
-; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB0_8: # %for.body
>From 8512a4ea7b158d111af2d3d801ff716fe9016449 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 7 Aug 2026 21:26:15 +0000
Subject: [PATCH 5/9] Adding/updating lit-tests
---
.../peephole-opt-beyond-cross-rc-copy.mir | 72 +++++++++++++++++++
llvm/test/CodeGen/AMDGPU/spill-vgpr.ll | 2 +-
2 files changed, 73 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
new file mode 100644
index 0000000000000..5cc68825c2f55
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
@@ -0,0 +1,72 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx950 -run-pass peephole-opt -verify-machineinstrs -o - %s | FileCheck %s
+
+# Reproduces the fp8_gemm pattern: an av_64_align2 COPY of a vreg_64_align2
+# REG_SEQUENCE whose sub0 is used cross-block. PeepholeOptimizer::findNextSource
+# should fold the bb.1 use through both the av COPY and the REG_SEQUENCE down to
+# the scalar source %0.
+
+---
+name: test
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF]], %subreg.sub0, [[DEF1]], %subreg.sub1
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_64_align2 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY1]]
+ bb.0:
+ successors: %bb.1
+
+ %0:vgpr_32 = IMPLICIT_DEF
+ %1:vgpr_32 = IMPLICIT_DEF
+ %2:vreg_64_align2 = REG_SEQUENCE %0:vgpr_32, %subreg.sub0, %1:vgpr_32, %subreg.sub1
+ %3:av_64_align2 = COPY %2:vreg_64_align2
+ S_BRANCH %bb.1
+
+ bb.1:
+ %4:vgpr_32 = COPY %3.sub0:av_64_align2
+ S_ENDPGM 0, implicit %4
+...
+
+# Fallback case: after the suitable subregister source (%0.sub0), the deeper
+# trace reaches a physical register ($vgpr0). findNextSource must fall back to
+# the remembered suitable source instead of bailing out, so the bb.1 use folds
+# to %0.sub0 (dropping the av COPY) rather than staying on %3.sub0.
+
+---
+name: test_phys_fallback
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_phys_fallback
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY [[COPY]]
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[COPY2]]
+ bb.0:
+ successors: %bb.1
+ liveins: $vgpr0_vgpr1
+
+ %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+ %1:av_64_align2 = COPY %0:vreg_64_align2
+ S_BRANCH %bb.1
+
+ bb.1:
+ %2:vgpr_32 = COPY %1.sub0:av_64_align2
+ S_ENDPGM 0, implicit %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 05dba10e9d9ab..64e673af67e3a 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -137,7 +137,7 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
; GFX900: NumVgprs: 256
; GFX908: NumVgprs: 254
-; GFX900: ScratchSize: 132
+; GFX900: ScratchSize: 148
; GFX908: ScratchSize: 0
; GFX900: VGPRBlocks: 63
; GFX908: VGPRBlocks: 63
>From cfd9b25378d92f4f5b71e4dd501587a65d4497d8 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 7 Aug 2026 22:06:31 +0000
Subject: [PATCH 6/9] Cleanup comments and dbg msgs.
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 27 +++++++-------------------
1 file changed, 7 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index fca2a670ea974..83099fbd1e2c6 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1022,12 +1022,9 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
RegSubRegPair CurSrcPair = RegSubReg;
SmallVector<RegSubRegPair, 4> SrcToLook = {CurSrcPair};
- LLVM_DEBUG(dbgs() << "EXPenter reg="
- << printReg(RegSubReg.Reg, TRI, RegSubReg.SubReg) << "\n");
unsigned PHICount = 0;
- // EXPERIMENT: remember the last suitable source so we can keep tracing past
- // it (through REG_SEQUENCE, etc.) and still fall back if the deeper trace
- // dead-ends.
+ // Remember the last suitable source so we can keep tracing past it (through
+ // REG_SEQUENCE, etc.) and still fall back if the deeper trace dead-ends.
bool FoundSuitable = false;
RegSubRegPair SuitablePair = RegSubReg;
// Fall back to the last suitable (sub)register source we remembered while
@@ -1058,13 +1055,9 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
ValueTrackerResult Res = ValTracker.getNextSource();
// Abort at the end of a chain (without finding a suitable source).
if (!Res.isValid()) {
- // EXPERIMENT: if we already passed a suitable source while trying to
- // reach a deeper one, fall back to it instead of aborting.
+ // If we already passed a suitable source while trying to reach a deeper
+ // one, fall back to it instead of aborting.
if (FoundSuitable) {
- LLVM_DEBUG(dbgs()
- << "EXP fallback "
- << printReg(SuitablePair.Reg, TRI, SuitablePair.SubReg)
- << " (orig " << printReg(Reg, TRI) << ")\n");
FallBackToSuitable();
break;
}
@@ -1135,22 +1128,16 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
if (PHICount > 0 && CurSrcPair.SubReg != 0)
continue;
- // EXPERIMENT: don't stop at the first suitable source if it is still a
- // subregister; keep tracing to try to reach a deeper source. Remember it
- // as a fallback.
+ // Don't stop at the first suitable source if it is still a subregister;
+ // keep tracing to try to reach a deeper source. Remember it as a
+ // fallback.
if (CurSrcPair.SubReg != 0) {
- LLVM_DEBUG(dbgs() << "EXP continue "
- << printReg(CurSrcPair.Reg, TRI, CurSrcPair.SubReg)
- << " (orig " << printReg(Reg, TRI) << ")\n");
SuitablePair = CurSrcPair;
FoundSuitable = true;
continue;
}
// We found a suitable source, and are done with this chain.
- LLVM_DEBUG(dbgs() << "EXP final "
- << printReg(CurSrcPair.Reg, TRI, CurSrcPair.SubReg)
- << " (orig " << printReg(Reg, TRI) << ")\n");
break;
}
} while (!SrcToLook.empty());
>From 7dc4a2574cfed960fd83a193221649be0c14a2a6 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonseo.choi at amd.com>
Date: Fri, 7 Aug 2026 23:26:31 +0000
Subject: [PATCH 7/9] Resolve rebase conflicts with upstream main
---
llvm/test/CodeGen/AMDGPU/div_i128.ll | 167 +++--
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 942 ++++++++++++-------------
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 170 ++---
3 files changed, 634 insertions(+), 645 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index b38c24f6e8eb4..f7932ad494b72 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -9,26 +9,26 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-LABEL: v_sdiv_i128_vv:
; GFX9: ; %bb.0: ; %_udiv-special-cases
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i32_e32 v16, 31, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v16
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v16
-; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v16
-; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v16, vcc
-; GFX9-NEXT: v_xor_b32_e32 v1, v2, v16
-; GFX9-NEXT: v_ashrrev_i32_e32 v17, 31, v7
-; GFX9-NEXT: v_xor_b32_e32 v0, v3, v16
-; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v1, v16, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v0, v16, vcc
-; GFX9-NEXT: v_xor_b32_e32 v1, v4, v17
-; GFX9-NEXT: v_xor_b32_e32 v0, v5, v17
-; GFX9-NEXT: v_sub_co_u32_e32 v20, vcc, v1, v17
-; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v0, v17, vcc
-; GFX9-NEXT: v_xor_b32_e32 v0, v6, v17
-; GFX9-NEXT: v_xor_b32_e32 v1, v7, v17
-; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v17, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v17, vcc
-; GFX9-NEXT: v_or_b32_e32 v3, v21, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v20, v0
+; GFX9-NEXT: v_ashrrev_i32_e32 v14, 31, v3
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v14
+; GFX9-NEXT: v_xor_b32_e32 v1, v1, v14
+; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v0, v14
+; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v1, v14, vcc
+; GFX9-NEXT: v_xor_b32_e32 v1, v2, v14
+; GFX9-NEXT: v_ashrrev_i32_e32 v15, 31, v7
+; GFX9-NEXT: v_xor_b32_e32 v0, v3, v14
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v1, v14, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v0, v14, vcc
+; GFX9-NEXT: v_xor_b32_e32 v1, v4, v15
+; GFX9-NEXT: v_xor_b32_e32 v0, v5, v15
+; GFX9-NEXT: v_sub_co_u32_e32 v16, vcc, v1, v15
+; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v0, v15, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v6, v15
+; GFX9-NEXT: v_xor_b32_e32 v1, v7, v15
+; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v15, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v15, vcc
+; GFX9-NEXT: v_or_b32_e32 v3, v17, v1
+; GFX9-NEXT: v_or_b32_e32 v2, v16, v0
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
; GFX9-NEXT: v_or_b32_e32 v3, v9, v11
; GFX9-NEXT: v_or_b32_e32 v2, v8, v10
@@ -37,9 +37,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_u32_e32 v2, 32, v2
; GFX9-NEXT: v_ffbh_u32_e32 v3, v1
; GFX9-NEXT: v_min_u32_e32 v2, v2, v3
-; GFX9-NEXT: v_ffbh_u32_e32 v3, v20
+; GFX9-NEXT: v_ffbh_u32_e32 v3, v16
; GFX9-NEXT: v_add_u32_e32 v3, 32, v3
-; GFX9-NEXT: v_ffbh_u32_e32 v4, v21
+; GFX9-NEXT: v_ffbh_u32_e32 v4, v17
; GFX9-NEXT: v_min_u32_e32 v3, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, 64, v3
@@ -58,7 +58,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, 64, v5
; GFX9-NEXT: v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX9-NEXT: v_mov_b32_e32 v18, v16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
@@ -67,7 +66,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_subb_co_u32_e64 v5, s[6:7], 0, 0, s[6:7]
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v19, v17
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
@@ -76,8 +74,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_and_b32_e32 v6, 1, v6
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
; GFX9-NEXT: v_xor_b32_e32 v6, 0x7f, v2
-; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
; GFX9-NEXT: v_or_b32_e32 v7, v3, v5
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
@@ -89,11 +87,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v3, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v4, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v18, vcc, 1, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v3, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v20, vcc, 0, v4, vcc
; GFX9-NEXT: v_sub_u32_e32 v7, 0x7f, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v5, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v5, vcc
; GFX9-NEXT: v_sub_u32_e32 v5, 64, v7
; GFX9-NEXT: v_lshlrev_b64 v[3:4], v7, v[10:11]
; GFX9-NEXT: v_lshrrev_b64 v[5:6], v5, v[8:9]
@@ -117,71 +115,71 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
-; GFX9-NEXT: v_sub_u32_e32 v12, 64, v22
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], v22, v[8:9]
+; GFX9-NEXT: v_sub_u32_e32 v12, 64, v18
+; GFX9-NEXT: v_lshrrev_b64 v[6:7], v18, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[12:13], v12, v[10:11]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
; GFX9-NEXT: v_or_b32_e32 v12, v6, v12
-; GFX9-NEXT: v_subrev_u32_e32 v6, 64, v22
+; GFX9-NEXT: v_subrev_u32_e32 v6, 64, v18
; GFX9-NEXT: v_or_b32_e32 v13, v7, v13
; GFX9-NEXT: v_lshrrev_b64 v[6:7], v6, v[10:11]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v22
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v9, v7, v9, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v6, v12, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[6:7], v22, v[10:11]
+; GFX9-NEXT: v_lshrrev_b64 v[6:7], v18, v[10:11]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v12, v8, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v13, 0, v7, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v12, 0, v6, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v26, vcc, -1, v20
-; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v21, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v0, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v1, vcc
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v11, 0, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v10, 0, v6, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, -1, v16
+; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v17, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v0, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v1, vcc
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v14, 0
-; GFX9-NEXT: v_mov_b32_e32 v15, 0
+; GFX9-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: v_mov_b32_e32 v7, 0
; GFX9-NEXT: .LBB0_3: ; %udiv-do-while
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_lshlrev_b64 v[30:31], 1, v[4:5]
-; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13]
-; GFX9-NEXT: v_or_b32_e32 v4, v14, v30
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v9
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
-; GFX9-NEXT: v_or_b32_e32 v12, v12, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v3
-; GFX9-NEXT: v_or_b32_e32 v8, v8, v14
-; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v26, v8
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v27, v9, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v28, v12, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v29, v13, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v30, 31, v14
-; GFX9-NEXT: v_and_b32_e32 v14, v30, v20
; GFX9-NEXT: v_lshrrev_b32_e32 v6, 31, v5
-; GFX9-NEXT: v_or_b32_e32 v5, v15, v31
-; GFX9-NEXT: v_and_b32_e32 v15, v30, v21
-; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v14
-; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v9, v15, vcc
-; GFX9-NEXT: v_and_b32_e32 v14, v30, v0
-; GFX9-NEXT: v_and_b32_e32 v15, v30, v1
-; GFX9-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v14, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v13, v15, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, -1, v22
-; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc
+; GFX9-NEXT: v_lshlrev_b64 v[4:5], 1, v[4:5]
+; GFX9-NEXT: v_lshrrev_b32_e32 v26, 31, v3
; GFX9-NEXT: v_lshlrev_b64 v[2:3], 1, v[2:3]
-; GFX9-NEXT: v_or_b32_e32 v14, v22, v24
-; GFX9-NEXT: v_or_b32_e32 v15, v23, v25
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; GFX9-NEXT: v_or3_b32 v2, v2, v6, v10
-; GFX9-NEXT: v_and_b32_e32 v6, 1, v30
-; GFX9-NEXT: v_or3_b32 v3, v3, 0, v11
+; GFX9-NEXT: v_or_b32_e32 v4, v12, v4
+; GFX9-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-NEXT: v_or3_b32 v2, v2, v6, v12
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 31, v9
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11]
+; GFX9-NEXT: v_or_b32_e32 v8, v8, v26
+; GFX9-NEXT: v_or_b32_e32 v6, v10, v6
+; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, v22, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v23, v9, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v24, v6, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v25, v11, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v26, 31, v10
+; GFX9-NEXT: v_and_b32_e32 v10, v26, v16
+; GFX9-NEXT: v_and_b32_e32 v12, v26, v17
+; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v10
+; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v9, v12, vcc
+; GFX9-NEXT: v_and_b32_e32 v10, v26, v0
+; GFX9-NEXT: v_and_b32_e32 v12, v26, v1
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v6, v10, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v12, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v18, vcc, -1, v18
+; GFX9-NEXT: v_addc_co_u32_e32 v19, vcc, -1, v19, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v20, vcc, -1, v20, vcc
+; GFX9-NEXT: v_or_b32_e32 v5, v13, v5
+; GFX9-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v21, vcc, -1, v21, vcc
+; GFX9-NEXT: v_or3_b32 v3, v3, 0, v13
+; GFX9-NEXT: v_or_b32_e32 v12, v18, v20
+; GFX9-NEXT: v_or_b32_e32 v13, v19, v21
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
+; GFX9-NEXT: v_and_b32_e32 v6, 1, v26
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v15, v7
-; GFX9-NEXT: v_mov_b32_e32 v14, v6
+; GFX9-NEXT: v_mov_b32_e32 v13, v7
+; GFX9-NEXT: v_mov_b32_e32 v12, v6
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-NEXT: ; %bb.4: ; %Flow
@@ -196,15 +194,14 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or_b32_e32 v6, v6, v0
; GFX9-NEXT: .LBB0_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: v_xor_b32_e32 v2, v17, v16
-; GFX9-NEXT: v_xor_b32_e32 v3, v19, v18
-; GFX9-NEXT: v_xor_b32_e32 v0, v6, v2
+; GFX9-NEXT: v_xor_b32_e32 v3, v15, v14
+; GFX9-NEXT: v_xor_b32_e32 v0, v6, v3
; GFX9-NEXT: v_xor_b32_e32 v1, v7, v3
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX9-NEXT: v_xor_b32_e32 v5, v12, v2
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v3
+; GFX9-NEXT: v_xor_b32_e32 v2, v12, v3
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-NEXT: v_xor_b32_e32 v4, v13, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v2, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index b9a2c67c65020..3a2f21b7dc79f 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -6,26 +6,26 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-LABEL: v_sdiv_v2i128_vv:
; SDAG: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: v_ashrrev_i32_e32 v22, 31, v3
-; SDAG-NEXT: v_xor_b32_e32 v0, v0, v22
-; SDAG-NEXT: v_xor_b32_e32 v1, v1, v22
-; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v0, v22
-; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v1, v22, vcc
-; SDAG-NEXT: v_xor_b32_e32 v1, v2, v22
-; SDAG-NEXT: v_ashrrev_i32_e32 v23, 31, v11
-; SDAG-NEXT: v_xor_b32_e32 v0, v3, v22
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v1, v22, vcc
-; SDAG-NEXT: v_subb_u32_e32 v19, vcc, v0, v22, vcc
-; SDAG-NEXT: v_xor_b32_e32 v1, v8, v23
-; SDAG-NEXT: v_xor_b32_e32 v0, v9, v23
-; SDAG-NEXT: v_sub_i32_e32 v26, vcc, v1, v23
-; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v0, v23, vcc
-; SDAG-NEXT: v_xor_b32_e32 v1, v10, v23
-; SDAG-NEXT: v_xor_b32_e32 v0, v11, v23
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v1, v23, vcc
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v0, v23, vcc
-; SDAG-NEXT: v_or_b32_e32 v1, v27, v3
-; SDAG-NEXT: v_or_b32_e32 v0, v26, v2
+; SDAG-NEXT: v_ashrrev_i32_e32 v25, 31, v3
+; SDAG-NEXT: v_xor_b32_e32 v0, v0, v25
+; SDAG-NEXT: v_xor_b32_e32 v1, v1, v25
+; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v0, v25
+; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v1, v25, vcc
+; SDAG-NEXT: v_xor_b32_e32 v1, v2, v25
+; SDAG-NEXT: v_ashrrev_i32_e32 v26, 31, v11
+; SDAG-NEXT: v_xor_b32_e32 v0, v3, v25
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v1, v25, vcc
+; SDAG-NEXT: v_subb_u32_e32 v19, vcc, v0, v25, vcc
+; SDAG-NEXT: v_xor_b32_e32 v1, v8, v26
+; SDAG-NEXT: v_xor_b32_e32 v0, v9, v26
+; SDAG-NEXT: v_sub_i32_e32 v27, vcc, v1, v26
+; SDAG-NEXT: v_subb_u32_e32 v28, vcc, v0, v26, vcc
+; SDAG-NEXT: v_xor_b32_e32 v1, v10, v26
+; SDAG-NEXT: v_xor_b32_e32 v0, v11, v26
+; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v1, v26, vcc
+; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v0, v26, vcc
+; SDAG-NEXT: v_or_b32_e32 v1, v28, v3
+; SDAG-NEXT: v_or_b32_e32 v0, v27, v2
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; SDAG-NEXT: v_or_b32_e32 v1, v17, v19
; SDAG-NEXT: v_or_b32_e32 v0, v16, v18
@@ -34,9 +34,9 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_add_i32_e64 v0, s[6:7], 32, v0
; SDAG-NEXT: v_ffbh_u32_e32 v1, v3
; SDAG-NEXT: v_min_u32_e32 v0, v0, v1
-; SDAG-NEXT: v_ffbh_u32_e32 v1, v26
+; SDAG-NEXT: v_ffbh_u32_e32 v1, v27
; SDAG-NEXT: v_add_i32_e64 v1, s[6:7], 32, v1
-; SDAG-NEXT: v_ffbh_u32_e32 v8, v27
+; SDAG-NEXT: v_ffbh_u32_e32 v8, v28
; SDAG-NEXT: v_min_u32_e32 v1, v1, v8
; SDAG-NEXT: v_add_i32_e64 v1, s[6:7], 64, v1
; SDAG-NEXT: v_addc_u32_e64 v8, s[6:7], 0, 0, s[6:7]
@@ -71,7 +71,6 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v24, v22
; SDAG-NEXT: v_cndmask_b32_e64 v0, v1, v0, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v0, 1, v0
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0
@@ -81,16 +80,15 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v0, v18, 0, s[4:5]
; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v25, v23
; SDAG-NEXT: v_cndmask_b32_e64 v20, v17, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v21, v16, 0, s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB0_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT: v_add_i32_e32 v28, vcc, 1, v8
-; SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v9, vcc
-; SDAG-NEXT: v_addc_u32_e32 v30, vcc, 0, v10, vcc
-; SDAG-NEXT: v_addc_u32_e64 v31, s[4:5], 0, v11, vcc
+; SDAG-NEXT: v_add_i32_e32 v29, vcc, 1, v8
+; SDAG-NEXT: v_addc_u32_e32 v30, vcc, 0, v9, vcc
+; SDAG-NEXT: v_addc_u32_e32 v31, vcc, 0, v10, vcc
+; SDAG-NEXT: v_addc_u32_e64 v32, s[4:5], 0, v11, vcc
; SDAG-NEXT: v_sub_i32_e32 v11, vcc, s8, v8
; SDAG-NEXT: v_sub_i32_e32 v9, vcc, 64, v11
; SDAG-NEXT: v_lshl_b64 v[0:1], v[18:19], v11
@@ -115,72 +113,72 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB0_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v28
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v28
+; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v29
+; SDAG-NEXT: v_lshr_b64 v[10:11], v[16:17], v29
; SDAG-NEXT: v_lshl_b64 v[20:21], v[18:19], v20
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v28
-; SDAG-NEXT: v_or_b32_e32 v20, v10, v20
-; SDAG-NEXT: v_subrev_i32_e32 v10, vcc, 64, v28
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v29
+; SDAG-NEXT: v_or_b32_e32 v22, v10, v20
+; SDAG-NEXT: v_subrev_i32_e32 v10, vcc, 64, v29
; SDAG-NEXT: v_or_b32_e32 v21, v11, v21
; SDAG-NEXT: v_lshr_b64 v[10:11], v[18:19], v10
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v28
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v29
; SDAG-NEXT: v_cndmask_b32_e32 v11, v11, v21, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v17, v11, v17, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v20, v10, v20, vcc
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[18:19], v28
-; SDAG-NEXT: v_cndmask_b32_e64 v16, v20, v16, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v19, 0, v11, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v18, 0, v10, vcc
-; SDAG-NEXT: v_add_i32_e32 v32, vcc, -1, v26
-; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v27, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v20, v11, v17, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v17, v10, v22, vcc
+; SDAG-NEXT: v_lshr_b64 v[10:11], v[18:19], v29
+; SDAG-NEXT: v_cndmask_b32_e64 v19, v17, v16, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v22, 0, v11, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v21, 0, v10, vcc
+; SDAG-NEXT: v_add_i32_e32 v18, vcc, -1, v27
+; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v28, vcc
; SDAG-NEXT: v_addc_u32_e32 v34, vcc, -1, v2, vcc
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v3, vcc
+; SDAG-NEXT: v_mov_b32_e32 v16, 0
+; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
-; SDAG-NEXT: v_mov_b32_e32 v21, 0
+; SDAG-NEXT: v_mov_b32_e32 v23, 0
+; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v11, 0
; SDAG-NEXT: .LBB0_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_lshl_b64 v[36:37], v[8:9], 1
-; SDAG-NEXT: v_lshl_b64 v[18:19], v[18:19], 1
-; SDAG-NEXT: v_or_b32_e32 v8, v20, v36
-; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v17
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; SDAG-NEXT: v_or_b32_e32 v18, v18, v20
-; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v1
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v20
-; SDAG-NEXT: v_sub_i32_e32 v20, vcc, v32, v16
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v33, v17, vcc
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v34, v18, vcc
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v35, v19, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v36, 31, v20
+; SDAG-NEXT: v_lshl_b64 v[21:22], v[21:22], 1
+; SDAG-NEXT: v_or_b32_e32 v8, v23, v36
+; SDAG-NEXT: v_lshrrev_b32_e32 v23, 31, v20
+; SDAG-NEXT: v_lshl_b64 v[19:20], v[19:20], 1
+; SDAG-NEXT: v_or_b32_e32 v21, v21, v23
+; SDAG-NEXT: v_lshrrev_b32_e32 v23, 31, v1
+; SDAG-NEXT: v_or_b32_e32 v19, v19, v23
+; SDAG-NEXT: v_sub_i32_e32 v23, vcc, v18, v19
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v33, v20, vcc
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v34, v21, vcc
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v35, v22, vcc
+; SDAG-NEXT: v_ashrrev_i32_e32 v36, 31, v23
+; SDAG-NEXT: v_and_b32_e32 v23, v36, v27
; SDAG-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; SDAG-NEXT: v_or_b32_e32 v9, v21, v37
-; SDAG-NEXT: v_and_b32_e32 v21, v36, v26
-; SDAG-NEXT: v_and_b32_e32 v20, v36, v27
-; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v16, v21
-; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v17, v20, vcc
-; SDAG-NEXT: v_and_b32_e32 v21, v36, v2
-; SDAG-NEXT: v_and_b32_e32 v20, v36, v3
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v18, v21, vcc
-; SDAG-NEXT: v_subb_u32_e32 v19, vcc, v19, v20, vcc
-; SDAG-NEXT: v_add_i32_e32 v28, vcc, -1, v28
-; SDAG-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
+; SDAG-NEXT: v_or_b32_e32 v9, v24, v37
+; SDAG-NEXT: v_and_b32_e32 v24, v36, v28
+; SDAG-NEXT: v_sub_i32_e32 v19, vcc, v19, v23
+; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v20, v24, vcc
+; SDAG-NEXT: v_and_b32_e32 v23, v36, v2
+; SDAG-NEXT: v_and_b32_e32 v24, v36, v3
+; SDAG-NEXT: v_subb_u32_e32 v21, vcc, v21, v23, vcc
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v22, v24, vcc
+; SDAG-NEXT: v_add_i32_e32 v29, vcc, -1, v29
; SDAG-NEXT: v_addc_u32_e32 v30, vcc, -1, v30, vcc
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
-; SDAG-NEXT: v_or_b32_e32 v21, v29, v31
-; SDAG-NEXT: v_or_b32_e32 v20, v28, v30
+; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
+; SDAG-NEXT: v_or_b32_e32 v23, v29, v31
+; SDAG-NEXT: v_or_b32_e32 v24, v30, v32
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[20:21]
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
-; SDAG-NEXT: v_mov_b32_e32 v21, 0
+; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[23:24]
; SDAG-NEXT: v_or_b32_e32 v0, v0, v10
; SDAG-NEXT: v_and_b32_e32 v10, 1, v36
-; SDAG-NEXT: v_or_b32_e32 v1, v21, v1
-; SDAG-NEXT: v_or_b32_e32 v0, v20, v0
+; SDAG-NEXT: v_or_b32_e32 v1, v17, v1
+; SDAG-NEXT: v_or_b32_e32 v0, v16, v0
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v21, v11
-; SDAG-NEXT: v_mov_b32_e32 v20, v10
+; SDAG-NEXT: v_mov_b32_e32 v24, v11
+; SDAG-NEXT: v_mov_b32_e32 v23, v10
; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_cbranch_execnz .LBB0_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
@@ -195,214 +193,210 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_or_b32_e32 v21, v10, v2
; SDAG-NEXT: .LBB0_6: ; %Flow16
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v7
-; SDAG-NEXT: v_xor_b32_e32 v3, v4, v16
-; SDAG-NEXT: v_xor_b32_e32 v2, v5, v16
-; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v3, v16
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v2, v16, vcc
-; SDAG-NEXT: v_xor_b32_e32 v3, v6, v16
-; SDAG-NEXT: v_ashrrev_i32_e32 v17, 31, v15
-; SDAG-NEXT: v_xor_b32_e32 v2, v7, v16
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v3, v16, vcc
-; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v2, v16, vcc
-; SDAG-NEXT: v_xor_b32_e32 v3, v12, v17
-; SDAG-NEXT: v_xor_b32_e32 v2, v13, v17
-; SDAG-NEXT: v_sub_i32_e32 v26, vcc, v3, v17
-; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v2, v17, vcc
-; SDAG-NEXT: v_xor_b32_e32 v2, v14, v17
-; SDAG-NEXT: v_xor_b32_e32 v3, v15, v17
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v17, vcc
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v17, vcc
-; SDAG-NEXT: v_or_b32_e32 v5, v27, v3
-; SDAG-NEXT: v_or_b32_e32 v4, v26, v2
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
-; SDAG-NEXT: v_or_b32_e32 v5, v9, v7
-; SDAG-NEXT: v_or_b32_e32 v4, v8, v6
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[4:5]
-; SDAG-NEXT: v_ffbh_u32_e32 v4, v2
+; SDAG-NEXT: v_ashrrev_i32_e32 v18, 31, v7
+; SDAG-NEXT: v_xor_b32_e32 v3, v4, v18
+; SDAG-NEXT: v_xor_b32_e32 v2, v5, v18
+; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v3, v18
+; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v2, v18, vcc
+; SDAG-NEXT: v_xor_b32_e32 v3, v6, v18
+; SDAG-NEXT: v_ashrrev_i32_e32 v19, 31, v15
+; SDAG-NEXT: v_xor_b32_e32 v2, v7, v18
+; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v3, v18, vcc
+; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v2, v18, vcc
+; SDAG-NEXT: v_xor_b32_e32 v3, v12, v19
+; SDAG-NEXT: v_xor_b32_e32 v2, v13, v19
+; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v3, v19
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v2, v19, vcc
+; SDAG-NEXT: v_xor_b32_e32 v2, v14, v19
+; SDAG-NEXT: v_xor_b32_e32 v3, v15, v19
+; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v19, vcc
+; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v19, vcc
+; SDAG-NEXT: v_or_b32_e32 v9, v23, v3
+; SDAG-NEXT: v_or_b32_e32 v8, v22, v2
+; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; SDAG-NEXT: v_or_b32_e32 v9, v5, v7
+; SDAG-NEXT: v_or_b32_e32 v8, v4, v6
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
+; SDAG-NEXT: v_ffbh_u32_e32 v8, v2
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_add_i32_e32 v4, vcc, 32, v4
-; SDAG-NEXT: v_ffbh_u32_e32 v5, v3
-; SDAG-NEXT: v_min_u32_e32 v4, v4, v5
-; SDAG-NEXT: v_ffbh_u32_e32 v5, v26
-; SDAG-NEXT: v_add_i32_e32 v5, vcc, 32, v5
-; SDAG-NEXT: v_ffbh_u32_e32 v10, v27
-; SDAG-NEXT: v_min_u32_e32 v5, v5, v10
-; SDAG-NEXT: v_add_i32_e32 v5, vcc, 64, v5
+; SDAG-NEXT: v_add_i32_e32 v8, vcc, 32, v8
+; SDAG-NEXT: v_ffbh_u32_e32 v9, v3
+; SDAG-NEXT: v_min_u32_e32 v8, v8, v9
+; SDAG-NEXT: v_ffbh_u32_e32 v9, v22
+; SDAG-NEXT: v_add_i32_e32 v9, vcc, 32, v9
+; SDAG-NEXT: v_ffbh_u32_e32 v10, v23
+; SDAG-NEXT: v_min_u32_e32 v9, v9, v10
+; SDAG-NEXT: v_add_i32_e32 v9, vcc, 64, v9
; SDAG-NEXT: v_addc_u32_e64 v10, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; SDAG-NEXT: v_ffbh_u32_e32 v11, v7
-; SDAG-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
-; SDAG-NEXT: v_ffbh_u32_e32 v5, v6
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
+; SDAG-NEXT: v_ffbh_u32_e32 v9, v6
; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; SDAG-NEXT: v_add_i32_e32 v5, vcc, 32, v5
-; SDAG-NEXT: v_min_u32_e32 v5, v5, v11
-; SDAG-NEXT: v_ffbh_u32_e32 v11, v8
+; SDAG-NEXT: v_add_i32_e32 v9, vcc, 32, v9
+; SDAG-NEXT: v_min_u32_e32 v9, v9, v11
+; SDAG-NEXT: v_ffbh_u32_e32 v11, v4
; SDAG-NEXT: v_add_i32_e32 v11, vcc, 32, v11
-; SDAG-NEXT: v_ffbh_u32_e32 v12, v9
+; SDAG-NEXT: v_ffbh_u32_e32 v12, v5
; SDAG-NEXT: v_min_u32_e32 v11, v11, v12
; SDAG-NEXT: v_add_i32_e32 v11, vcc, 64, v11
; SDAG-NEXT: v_addc_u32_e64 v12, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v18, v16
-; SDAG-NEXT: v_cndmask_b32_e32 v5, v11, v5, vcc
+; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v5
-; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v10, v12, vcc
-; SDAG-NEXT: v_subb_u32_e64 v12, s[6:7], 0, 0, vcc
-; SDAG-NEXT: v_subb_u32_e64 v13, s[6:7], 0, 0, s[6:7]
+; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v8, v9
+; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v10, v12, vcc
+; SDAG-NEXT: v_subb_u32_e64 v10, s[6:7], 0, 0, vcc
+; SDAG-NEXT: v_subb_u32_e64 v11, s[6:7], 0, 0, s[6:7]
; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[4:5]
-; SDAG-NEXT: v_or_b32_e32 v15, v5, v13
-; SDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT: v_mov_b32_e32 v19, v17
-; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
-; SDAG-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
-; SDAG-NEXT: v_and_b32_e32 v10, 1, v10
-; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
-; SDAG-NEXT: v_xor_b32_e32 v10, 0x7f, v4
-; SDAG-NEXT: v_or_b32_e32 v14, v10, v12
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; SDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; SDAG-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
+; SDAG-NEXT: v_and_b32_e32 v12, 1, v12
+; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12
+; SDAG-NEXT: v_xor_b32_e32 v12, 0x7f, v8
+; SDAG-NEXT: v_or_b32_e32 v13, v9, v11
+; SDAG-NEXT: v_or_b32_e32 v12, v12, v10
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
+; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v7, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v6, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v14, v9, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v15, v8, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v7, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v6, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v13, v5, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v12, v4, 0, s[4:5]
; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB0_12
; SDAG-NEXT: ; %bb.7: ; %udiv-bb1
-; SDAG-NEXT: v_add_i32_e32 v28, vcc, 1, v4
-; SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v5, vcc
-; SDAG-NEXT: v_addc_u32_e32 v30, vcc, 0, v12, vcc
-; SDAG-NEXT: v_addc_u32_e64 v31, s[4:5], 0, v13, vcc
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, s8, v4
-; SDAG-NEXT: v_sub_i32_e32 v5, vcc, 64, v14
-; SDAG-NEXT: v_lshl_b64 v[10:11], v[6:7], v14
-; SDAG-NEXT: v_lshr_b64 v[12:13], v[8:9], v5
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, 63, v4
-; SDAG-NEXT: v_lshl_b64 v[4:5], v[8:9], v4
-; SDAG-NEXT: v_or_b32_e32 v11, v11, v13
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: v_add_i32_e32 v24, vcc, 1, v8
+; SDAG-NEXT: v_addc_u32_e32 v27, vcc, 0, v9, vcc
+; SDAG-NEXT: v_addc_u32_e32 v28, vcc, 0, v10, vcc
+; SDAG-NEXT: v_addc_u32_e64 v29, s[4:5], 0, v11, vcc
+; SDAG-NEXT: v_sub_i32_e32 v13, vcc, s8, v8
+; SDAG-NEXT: v_sub_i32_e32 v11, vcc, 64, v13
+; SDAG-NEXT: v_lshl_b64 v[9:10], v[6:7], v13
+; SDAG-NEXT: v_lshr_b64 v[11:12], v[4:5], v11
+; SDAG-NEXT: v_sub_i32_e32 v8, vcc, 63, v8
+; SDAG-NEXT: v_or_b32_e32 v11, v9, v11
+; SDAG-NEXT: v_lshl_b64 v[8:9], v[4:5], v8
; SDAG-NEXT: v_or_b32_e32 v10, v10, v12
-; SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v11, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v5, v7, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v10, v4, v10, vcc
-; SDAG-NEXT: v_lshl_b64 v[4:5], v[8:9], v14
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v10, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc
+; SDAG-NEXT: v_lshl_b64 v[10:11], v[4:5], v13
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v7, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v6, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: v_mov_b32_e32 v13, 0
-; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v10, vcc
; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB0_11
; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v28
-; SDAG-NEXT: v_lshr_b64 v[12:13], v[8:9], v28
+; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v24
+; SDAG-NEXT: v_lshr_b64 v[12:13], v[4:5], v24
; SDAG-NEXT: v_lshl_b64 v[14:15], v[6:7], v14
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v28
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v24
; SDAG-NEXT: v_or_b32_e32 v14, v12, v14
-; SDAG-NEXT: v_subrev_i32_e32 v12, vcc, 64, v28
+; SDAG-NEXT: v_subrev_i32_e32 v12, vcc, 64, v24
; SDAG-NEXT: v_or_b32_e32 v15, v13, v15
; SDAG-NEXT: v_lshr_b64 v[12:13], v[6:7], v12
-; SDAG-NEXT: v_lshr_b64 v[6:7], v[6:7], v28
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v28
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
; SDAG-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v13, v5, s[4:5]
+; SDAG-NEXT: v_lshr_b64 v[5:6], v[6:7], v24
; SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v15, 0, v7, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v14, 0, v6, vcc
-; SDAG-NEXT: v_add_i32_e32 v32, vcc, -1, v26
-; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v27, vcc
-; SDAG-NEXT: v_addc_u32_e32 v34, vcc, -1, v2, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v9, v13, v9, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v8, v12, v8, s[4:5]
-; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v3, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v6, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v5, vcc
+; SDAG-NEXT: v_add_i32_e32 v30, vcc, -1, v22
+; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v23, vcc
+; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v2, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v12, v4, s[4:5]
+; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v3, vcc
+; SDAG-NEXT: v_mov_b32_e32 v4, 0
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b64 s[10:11], 0
-; SDAG-NEXT: v_mov_b32_e32 v6, 0
-; SDAG-NEXT: v_mov_b32_e32 v7, 0
+; SDAG-NEXT: v_mov_b32_e32 v16, 0
+; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: v_mov_b32_e32 v13, 0
; SDAG-NEXT: .LBB0_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[36:37], v[4:5], 1
-; SDAG-NEXT: v_add_i32_e32 v28, vcc, -1, v28
-; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v5
-; SDAG-NEXT: v_or_b32_e32 v5, v7, v37
-; SDAG-NEXT: v_or_b32_e32 v4, v6, v36
-; SDAG-NEXT: v_lshl_b64 v[6:7], v[8:9], 1
-; SDAG-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
-; SDAG-NEXT: v_lshrrev_b32_e32 v36, 31, v9
-; SDAG-NEXT: v_lshrrev_b32_e32 v37, 31, v11
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[10:11], 1
-; SDAG-NEXT: v_addc_u32_e32 v30, vcc, -1, v30, vcc
-; SDAG-NEXT: v_or_b32_e32 v6, v6, v37
-; SDAG-NEXT: v_mov_b32_e32 v11, 0
-; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v31, vcc
+; SDAG-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v15
; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT: v_or_b32_e32 v11, v11, v9
-; SDAG-NEXT: v_sub_i32_e32 v9, vcc, v32, v6
-; SDAG-NEXT: v_mov_b32_e32 v10, 0
+; SDAG-NEXT: v_or_b32_e32 v6, v6, v12
+; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v9
+; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
+; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v11
; SDAG-NEXT: v_or_b32_e32 v8, v8, v12
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v33, v7, vcc
-; SDAG-NEXT: v_or_b32_e32 v14, v14, v36
-; SDAG-NEXT: v_or_b32_e32 v10, v10, v8
-; SDAG-NEXT: v_or_b32_e32 v8, v28, v30
-; SDAG-NEXT: v_or_b32_e32 v9, v29, v31
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v34, v14, vcc
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v35, v15, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v36, 31, v8
-; SDAG-NEXT: v_and_b32_e32 v8, v36, v26
-; SDAG-NEXT: v_and_b32_e32 v9, v36, v27
-; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v6, v8
-; SDAG-NEXT: v_and_b32_e32 v12, 1, v36
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v7, v9, vcc
-; SDAG-NEXT: v_and_b32_e32 v37, v36, v3
-; SDAG-NEXT: v_and_b32_e32 v36, v36, v2
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, v14, v36, vcc
+; SDAG-NEXT: v_sub_i32_e32 v12, vcc, v30, v14
+; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v31, v15, vcc
+; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
+; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v32, v6, vcc
+; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v33, v7, vcc
+; SDAG-NEXT: v_add_i32_e64 v24, s[4:5], -1, v24
+; SDAG-NEXT: v_or_b32_e32 v10, v16, v10
+; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v12
+; SDAG-NEXT: v_addc_u32_e64 v27, s[4:5], -1, v27, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v11, v17, v11
+; SDAG-NEXT: v_and_b32_e32 v12, 1, v16
+; SDAG-NEXT: v_and_b32_e32 v34, v16, v3
+; SDAG-NEXT: v_and_b32_e32 v35, v16, v2
+; SDAG-NEXT: v_and_b32_e32 v17, v16, v23
+; SDAG-NEXT: v_and_b32_e32 v16, v16, v22
+; SDAG-NEXT: v_addc_u32_e64 v28, s[4:5], -1, v28, s[4:5]
+; SDAG-NEXT: v_sub_i32_e32 v14, vcc, v14, v16
+; SDAG-NEXT: v_addc_u32_e64 v29, s[4:5], -1, v29, s[4:5]
+; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v15, v17, vcc
+; SDAG-NEXT: v_or_b32_e32 v17, v27, v29
+; SDAG-NEXT: v_or_b32_e32 v16, v24, v28
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[16:17]
+; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v35, vcc
+; SDAG-NEXT: v_or_b32_e32 v9, v5, v9
+; SDAG-NEXT: v_or_b32_e32 v8, v4, v8
+; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v34, vcc
; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v6, v12
-; SDAG-NEXT: v_mov_b32_e32 v7, v13
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v15, v37, vcc
+; SDAG-NEXT: v_mov_b32_e32 v17, v13
+; SDAG-NEXT: v_mov_b32_e32 v16, v12
; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB0_9
; SDAG-NEXT: ; %bb.10: ; %Flow
; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: .LBB0_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[4:5], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v5
-; SDAG-NEXT: v_or_b32_e32 v10, v10, v6
-; SDAG-NEXT: v_or_b32_e32 v14, v13, v3
-; SDAG-NEXT: v_or_b32_e32 v15, v12, v2
+; SDAG-NEXT: v_lshl_b64 v[14:15], v[8:9], 1
+; SDAG-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v4, 31, v11
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v4
+; SDAG-NEXT: v_or_b32_e32 v13, v13, v3
+; SDAG-NEXT: v_or_b32_e32 v12, v12, v2
; SDAG-NEXT: .LBB0_12: ; %Flow12
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_xor_b32_e32 v2, v23, v22
-; SDAG-NEXT: v_xor_b32_e32 v3, v25, v24
-; SDAG-NEXT: v_xor_b32_e32 v5, v0, v2
-; SDAG-NEXT: v_xor_b32_e32 v0, v21, v2
+; SDAG-NEXT: v_xor_b32_e32 v3, v26, v25
+; SDAG-NEXT: v_xor_b32_e32 v2, v0, v3
+; SDAG-NEXT: v_xor_b32_e32 v0, v21, v3
; SDAG-NEXT: v_xor_b32_e32 v4, v1, v3
; SDAG-NEXT: v_xor_b32_e32 v1, v20, v3
-; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v5, v2, vcc
-; SDAG-NEXT: v_xor_b32_e32 v6, v17, v16
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v4, v3, vcc
+; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v3, vcc
; SDAG-NEXT: v_xor_b32_e32 v7, v19, v18
-; SDAG-NEXT: v_xor_b32_e32 v4, v15, v6
-; SDAG-NEXT: v_xor_b32_e32 v5, v14, v7
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
-; SDAG-NEXT: v_xor_b32_e32 v9, v10, v6
+; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v4, v3, vcc
+; SDAG-NEXT: v_xor_b32_e32 v4, v12, v7
+; SDAG-NEXT: v_xor_b32_e32 v5, v13, v7
+; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v7
+; SDAG-NEXT: v_xor_b32_e32 v6, v14, v7
; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v7, vcc
-; SDAG-NEXT: v_xor_b32_e32 v8, v11, v7
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v9, v6, vcc
+; SDAG-NEXT: v_xor_b32_e32 v8, v15, v7
+; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v7, vcc
; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v8, v7, vcc
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1689,7 +1683,6 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[8:9]
; SDAG-NEXT: v_cmp_eq_u64_e64 s[8:9], 0, v[18:19]
-; SDAG-NEXT: v_mov_b32_e32 v31, v28
; SDAG-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[8:9]
; SDAG-NEXT: v_and_b32_e32 v10, 1, v10
; SDAG-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v10
@@ -1699,15 +1692,15 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_movk_i32 s8, 0x7f
; SDAG-NEXT: v_cndmask_b32_e64 v10, v2, 0, s[4:5]
; SDAG-NEXT: s_and_b64 s[10:11], s[10:11], s[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v33, v1, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v34, v1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v35, v0, 0, s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB4_6
; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT: v_add_i32_e32 v32, vcc, 1, v16
-; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v17, vcc
-; SDAG-NEXT: v_addc_u32_e32 v34, vcc, 0, v18, vcc
-; SDAG-NEXT: v_addc_u32_e64 v35, s[4:5], 0, v19, vcc
+; SDAG-NEXT: v_add_i32_e32 v31, vcc, 1, v16
+; SDAG-NEXT: v_addc_u32_e32 v32, vcc, 0, v17, vcc
+; SDAG-NEXT: v_addc_u32_e32 v33, vcc, 0, v18, vcc
+; SDAG-NEXT: v_addc_u32_e64 v34, s[4:5], 0, v19, vcc
; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s8, v16
; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
; SDAG-NEXT: v_lshl_b64 v[10:11], v[2:3], v19
@@ -1732,106 +1725,106 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB4_5
; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v32
-; SDAG-NEXT: v_lshr_b64 v[18:19], v[0:1], v32
+; SDAG-NEXT: v_sub_i32_e32 v20, vcc, 64, v31
+; SDAG-NEXT: v_lshr_b64 v[18:19], v[0:1], v31
; SDAG-NEXT: v_lshl_b64 v[20:21], v[2:3], v20
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v32
+; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v31
; SDAG-NEXT: v_or_b32_e32 v20, v18, v20
-; SDAG-NEXT: v_subrev_i32_e32 v18, vcc, 64, v32
+; SDAG-NEXT: v_subrev_i32_e32 v18, vcc, 64, v31
; SDAG-NEXT: v_or_b32_e32 v21, v19, v21
; SDAG-NEXT: v_lshr_b64 v[18:19], v[2:3], v18
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v32
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v31
; SDAG-NEXT: v_cndmask_b32_e32 v19, v19, v21, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v21, v19, v1, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v23, v19, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v20, v18, v20, vcc
-; SDAG-NEXT: v_lshr_b64 v[18:19], v[2:3], v32
-; SDAG-NEXT: v_cndmask_b32_e64 v20, v20, v0, s[4:5]
+; SDAG-NEXT: v_lshr_b64 v[18:19], v[2:3], v31
+; SDAG-NEXT: v_cndmask_b32_e64 v22, v20, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v25, 0, v19, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v24, 0, v18, vcc
-; SDAG-NEXT: v_add_i32_e32 v36, vcc, -1, v30
-; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v29, vcc
-; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v8, vcc
-; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v9, vcc
-; SDAG-NEXT: v_mov_b32_e32 v22, 0
-; SDAG-NEXT: v_mov_b32_e32 v23, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
+; SDAG-NEXT: v_add_i32_e32 v35, vcc, -1, v30
+; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v29, vcc
+; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v8, vcc
+; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v9, vcc
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0
+; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v26, 0
; SDAG-NEXT: v_mov_b32_e32 v27, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: .LBB4_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[48:49], v[16:17], 1
-; SDAG-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
-; SDAG-NEXT: v_or_b32_e32 v16, v26, v48
-; SDAG-NEXT: v_lshrrev_b32_e32 v26, 31, v21
-; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
-; SDAG-NEXT: v_or_b32_e32 v24, v24, v26
-; SDAG-NEXT: v_lshrrev_b32_e32 v26, 31, v11
-; SDAG-NEXT: v_or_b32_e32 v20, v20, v26
-; SDAG-NEXT: v_sub_i32_e32 v26, vcc, v36, v20
-; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v37, v21, vcc
-; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v38, v24, vcc
-; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v39, v25, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v48, 31, v26
; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v17
-; SDAG-NEXT: v_or_b32_e32 v17, v27, v49
-; SDAG-NEXT: v_and_b32_e32 v27, v48, v30
-; SDAG-NEXT: v_and_b32_e32 v26, v48, v29
-; SDAG-NEXT: v_sub_i32_e32 v20, vcc, v20, v27
-; SDAG-NEXT: v_subb_u32_e32 v21, vcc, v21, v26, vcc
-; SDAG-NEXT: v_and_b32_e32 v27, v48, v8
-; SDAG-NEXT: v_and_b32_e32 v26, v48, v9
-; SDAG-NEXT: v_subb_u32_e32 v24, vcc, v24, v27, vcc
-; SDAG-NEXT: v_subb_u32_e32 v25, vcc, v25, v26, vcc
-; SDAG-NEXT: v_add_i32_e32 v32, vcc, -1, v32
+; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
+; SDAG-NEXT: v_add_i32_e32 v31, vcc, -1, v31
+; SDAG-NEXT: v_or_b32_e32 v16, v26, v16
+; SDAG-NEXT: v_lshrrev_b32_e32 v26, 31, v23
+; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
+; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v32, vcc
+; SDAG-NEXT: v_or_b32_e32 v17, v27, v17
+; SDAG-NEXT: v_lshrrev_b32_e32 v27, 31, v11
+; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v33, vcc
+; SDAG-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
+; SDAG-NEXT: v_or_b32_e32 v22, v22, v27
; SDAG-NEXT: v_addc_u32_e32 v34, vcc, -1, v34, vcc
-; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v35, vcc
-; SDAG-NEXT: v_or_b32_e32 v27, v33, v35
-; SDAG-NEXT: v_or_b32_e32 v26, v32, v34
-; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[26:27]
; SDAG-NEXT: v_or_b32_e32 v10, v10, v18
-; SDAG-NEXT: v_and_b32_e32 v18, 1, v48
-; SDAG-NEXT: v_or_b32_e32 v11, v23, v11
-; SDAG-NEXT: v_or_b32_e32 v10, v22, v10
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v35, v22
+; SDAG-NEXT: v_or_b32_e32 v24, v24, v26
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v36, v23, vcc
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v37, v24, vcc
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v38, v25, vcc
+; SDAG-NEXT: v_or_b32_e32 v26, v31, v33
+; SDAG-NEXT: v_or_b32_e32 v27, v32, v34
+; SDAG-NEXT: v_ashrrev_i32_e32 v18, 31, v18
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[26:27]
+; SDAG-NEXT: v_and_b32_e32 v27, v18, v30
+; SDAG-NEXT: v_and_b32_e32 v26, v18, v29
+; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v22, v27
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v26, vcc
+; SDAG-NEXT: v_and_b32_e32 v48, v18, v8
+; SDAG-NEXT: v_and_b32_e32 v39, v18, v9
+; SDAG-NEXT: v_and_b32_e32 v18, 1, v18
+; SDAG-NEXT: v_subb_u32_e32 v24, vcc, v24, v48, vcc
+; SDAG-NEXT: v_or_b32_e32 v11, v21, v11
+; SDAG-NEXT: v_or_b32_e32 v10, v20, v10
+; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
; SDAG-NEXT: v_mov_b32_e32 v27, v19
; SDAG-NEXT: v_mov_b32_e32 v26, v18
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SDAG-NEXT: v_subb_u32_e32 v25, vcc, v25, v39, vcc
+; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB4_3
; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: .LBB4_5: ; %Flow14
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v17
; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; SDAG-NEXT: v_or_b32_e32 v10, v10, v20
-; SDAG-NEXT: v_or_b32_e32 v33, v19, v17
+; SDAG-NEXT: v_or_b32_e32 v34, v19, v17
; SDAG-NEXT: v_or_b32_e32 v35, v18, v16
; SDAG-NEXT: .LBB4_6: ; %Flow16
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_ashrrev_i32_e32 v26, 31, v7
-; SDAG-NEXT: v_xor_b32_e32 v4, v4, v26
-; SDAG-NEXT: v_xor_b32_e32 v5, v5, v26
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v26
-; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v26, vcc
-; SDAG-NEXT: v_xor_b32_e32 v6, v6, v26
-; SDAG-NEXT: v_xor_b32_e32 v7, v7, v26
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v26, vcc
+; SDAG-NEXT: v_ashrrev_i32_e32 v31, 31, v7
+; SDAG-NEXT: v_xor_b32_e32 v4, v4, v31
+; SDAG-NEXT: v_xor_b32_e32 v5, v5, v31
+; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v31
+; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v31, vcc
+; SDAG-NEXT: v_xor_b32_e32 v6, v6, v31
+; SDAG-NEXT: v_xor_b32_e32 v7, v7, v31
+; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v31, vcc
; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v26, vcc
+; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v31, vcc
; SDAG-NEXT: v_xor_b32_e32 v12, v12, v16
; SDAG-NEXT: v_xor_b32_e32 v13, v13, v16
-; SDAG-NEXT: v_sub_i32_e32 v34, vcc, v12, v16
-; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v13, v16, vcc
+; SDAG-NEXT: v_sub_i32_e32 v33, vcc, v12, v16
+; SDAG-NEXT: v_subb_u32_e32 v32, vcc, v13, v16, vcc
; SDAG-NEXT: v_xor_b32_e32 v12, v14, v16
; SDAG-NEXT: v_xor_b32_e32 v13, v15, v16
; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v12, v16, vcc
; SDAG-NEXT: v_subb_u32_e32 v13, vcc, v13, v16, vcc
-; SDAG-NEXT: v_or_b32_e32 v15, v27, v13
-; SDAG-NEXT: v_or_b32_e32 v14, v34, v12
+; SDAG-NEXT: v_or_b32_e32 v15, v32, v13
+; SDAG-NEXT: v_or_b32_e32 v14, v33, v12
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
; SDAG-NEXT: v_or_b32_e32 v15, v5, v7
; SDAG-NEXT: v_or_b32_e32 v14, v4, v6
@@ -1841,81 +1834,80 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_add_i32_e32 v14, vcc, 32, v14
; SDAG-NEXT: v_ffbh_u32_e32 v15, v13
; SDAG-NEXT: v_min_u32_e32 v14, v14, v15
-; SDAG-NEXT: v_ffbh_u32_e32 v15, v34
+; SDAG-NEXT: v_ffbh_u32_e32 v15, v33
; SDAG-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; SDAG-NEXT: v_ffbh_u32_e32 v16, v27
+; SDAG-NEXT: v_ffbh_u32_e32 v16, v32
; SDAG-NEXT: v_min_u32_e32 v15, v15, v16
; SDAG-NEXT: v_add_i32_e32 v15, vcc, 64, v15
; SDAG-NEXT: v_addc_u32_e64 v16, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT: v_ffbh_u32_e32 v17, v7
+; SDAG-NEXT: v_ffbh_u32_e32 v18, v5
; SDAG-NEXT: v_cndmask_b32_e32 v14, v15, v14, vcc
; SDAG-NEXT: v_ffbh_u32_e32 v15, v6
-; SDAG-NEXT: v_cndmask_b32_e64 v16, v16, 0, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v17, v16, 0, vcc
; SDAG-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; SDAG-NEXT: v_min_u32_e32 v15, v15, v17
-; SDAG-NEXT: v_ffbh_u32_e32 v17, v4
-; SDAG-NEXT: v_add_i32_e32 v17, vcc, 32, v17
-; SDAG-NEXT: v_ffbh_u32_e32 v18, v5
-; SDAG-NEXT: v_min_u32_e32 v17, v17, v18
-; SDAG-NEXT: v_add_i32_e32 v17, vcc, 64, v17
+; SDAG-NEXT: v_ffbh_u32_e32 v16, v7
+; SDAG-NEXT: v_min_u32_e32 v15, v15, v16
+; SDAG-NEXT: v_ffbh_u32_e32 v16, v4
+; SDAG-NEXT: v_add_i32_e32 v16, vcc, 32, v16
+; SDAG-NEXT: v_min_u32_e32 v16, v16, v18
+; SDAG-NEXT: v_add_i32_e32 v16, vcc, 64, v16
; SDAG-NEXT: v_addc_u32_e64 v18, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v32, v26
-; SDAG-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc
+; SDAG-NEXT: s_movk_i32 s8, 0x7f
+; SDAG-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v18, v18, 0, vcc
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, v14, v15
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v16, v18, vcc
+; SDAG-NEXT: v_sub_i32_e32 v16, vcc, v14, v15
+; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v17, v18, vcc
; SDAG-NEXT: v_subb_u32_e64 v18, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_subb_u32_e64 v19, s[6:7], 0, 0, s[6:7]
; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[14:15]
-; SDAG-NEXT: v_or_b32_e32 v21, v15, v19
-; SDAG-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[16:17]
+; SDAG-NEXT: v_or_b32_e32 v21, v17, v19
+; SDAG-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[18:19]
-; SDAG-NEXT: s_movk_i32 s8, 0x7f
-; SDAG-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
-; SDAG-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc
-; SDAG-NEXT: v_and_b32_e32 v16, 1, v16
-; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v16
-; SDAG-NEXT: v_xor_b32_e32 v16, 0x7f, v14
-; SDAG-NEXT: v_or_b32_e32 v20, v16, v18
+; SDAG-NEXT: v_cndmask_b32_e32 v14, v15, v14, vcc
+; SDAG-NEXT: v_and_b32_e32 v14, 1, v14
+; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v14
+; SDAG-NEXT: v_xor_b32_e32 v14, 0x7f, v16
+; SDAG-NEXT: v_or_b32_e32 v20, v14, v18
; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[20:21]
; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v17, v7, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v16, v6, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v7, 0, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v6, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v20, v5, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v21, v4, 0, s[4:5]
; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB4_12
; SDAG-NEXT: ; %bb.7: ; %udiv-bb1
-; SDAG-NEXT: v_add_i32_e32 v36, vcc, 1, v14
-; SDAG-NEXT: v_addc_u32_e32 v37, vcc, 0, v15, vcc
+; SDAG-NEXT: v_add_i32_e32 v36, vcc, 1, v16
+; SDAG-NEXT: v_addc_u32_e32 v37, vcc, 0, v17, vcc
; SDAG-NEXT: v_addc_u32_e32 v38, vcc, 0, v18, vcc
; SDAG-NEXT: v_addc_u32_e64 v39, s[4:5], 0, v19, vcc
-; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s8, v14
+; SDAG-NEXT: v_sub_i32_e32 v19, vcc, s8, v16
; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
-; SDAG-NEXT: v_lshl_b64 v[15:16], v[6:7], v19
+; SDAG-NEXT: v_lshl_b64 v[14:15], v[6:7], v19
; SDAG-NEXT: v_lshr_b64 v[17:18], v[4:5], v17
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 63, v14
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v18
-; SDAG-NEXT: v_or_b32_e32 v18, v15, v17
+; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; SDAG-NEXT: v_or_b32_e32 v17, v14, v17
+; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 63, v16
+; SDAG-NEXT: v_or_b32_e32 v18, v15, v18
; SDAG-NEXT: v_lshl_b64 v[14:15], v[4:5], v14
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; SDAG-NEXT: s_xor_b64 s[8:9], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e32 v15, v15, v16, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc
+; SDAG-NEXT: v_lshl_b64 v[16:17], v[4:5], v19
+; SDAG-NEXT: v_cndmask_b32_e32 v15, v15, v18, vcc
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v19
-; SDAG-NEXT: v_cndmask_b32_e64 v17, v15, v7, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v16, v14, v18, vcc
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[4:5], v19
-; SDAG-NEXT: v_cndmask_b32_e64 v16, v16, v6, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v15, 0, v15, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v15, v15, v7, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v14, v14, v6, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v17, vcc
; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
-; SDAG-NEXT: v_cndmask_b32_e32 v14, 0, v14, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v16, vcc
; SDAG-NEXT: s_and_saveexec_b64 s[4:5], s[8:9]
; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; SDAG-NEXT: s_cbranch_execz .LBB4_11
@@ -1934,135 +1926,135 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v20, v18, v20, vcc
; SDAG-NEXT: v_lshr_b64 v[18:19], v[6:7], v36
; SDAG-NEXT: v_cndmask_b32_e64 v22, v20, v4, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v25, 0, v19, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v24, 0, v18, vcc
-; SDAG-NEXT: v_add_i32_e32 v48, vcc, -1, v34
-; SDAG-NEXT: v_addc_u32_e32 v49, vcc, -1, v27, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v27, 0, v19, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v26, 0, v18, vcc
+; SDAG-NEXT: v_add_i32_e32 v48, vcc, -1, v33
+; SDAG-NEXT: v_addc_u32_e32 v49, vcc, -1, v32, vcc
; SDAG-NEXT: v_addc_u32_e32 v50, vcc, -1, v12, vcc
; SDAG-NEXT: v_addc_u32_e32 v51, vcc, -1, v13, vcc
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
+; SDAG-NEXT: s_mov_b64 s[10:11], 0
+; SDAG-NEXT: v_mov_b32_e32 v24, 0
+; SDAG-NEXT: v_mov_b32_e32 v25, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: .LBB4_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_add_i32_e32 v36, vcc, -1, v36
+; SDAG-NEXT: v_lshl_b64 v[26:27], v[26:27], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v23
; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
-; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v37, vcc
-; SDAG-NEXT: v_lshl_b64 v[24:25], v[24:25], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v52, 31, v17
-; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v38, vcc
-; SDAG-NEXT: v_or_b32_e32 v22, v22, v52
-; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v39, vcc
-; SDAG-NEXT: v_or_b32_e32 v24, v24, v18
-; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v48, v22
-; SDAG-NEXT: v_lshrrev_b32_e32 v54, 31, v15
+; SDAG-NEXT: v_or_b32_e32 v26, v26, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v15
; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
+; SDAG-NEXT: v_or_b32_e32 v22, v22, v18
+; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v17
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v18
+; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v48, v22
; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v49, v23, vcc
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v50, v24, vcc
+; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v50, v26, vcc
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v51, v27, vcc
+; SDAG-NEXT: v_add_i32_e64 v36, s[4:5], -1, v36
+; SDAG-NEXT: v_or_b32_e32 v16, v24, v16
+; SDAG-NEXT: v_ashrrev_i32_e32 v24, 31, v18
+; SDAG-NEXT: v_addc_u32_e64 v37, s[4:5], -1, v37, s[4:5]
+; SDAG-NEXT: v_or_b32_e32 v17, v25, v17
+; SDAG-NEXT: v_and_b32_e32 v18, 1, v24
+; SDAG-NEXT: v_and_b32_e32 v52, v24, v13
+; SDAG-NEXT: v_and_b32_e32 v53, v24, v12
+; SDAG-NEXT: v_and_b32_e32 v25, v24, v32
+; SDAG-NEXT: v_and_b32_e32 v24, v24, v33
+; SDAG-NEXT: v_addc_u32_e64 v38, s[4:5], -1, v38, s[4:5]
+; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v22, v24
+; SDAG-NEXT: v_addc_u32_e64 v39, s[4:5], -1, v39, s[4:5]
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v25, vcc
+; SDAG-NEXT: v_or_b32_e32 v25, v37, v39
+; SDAG-NEXT: v_or_b32_e32 v24, v36, v38
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[24:25]
+; SDAG-NEXT: v_subb_u32_e32 v26, vcc, v26, v53, vcc
; SDAG-NEXT: v_or_b32_e32 v15, v21, v15
; SDAG-NEXT: v_or_b32_e32 v14, v20, v14
-; SDAG-NEXT: v_or_b32_e32 v20, v36, v38
-; SDAG-NEXT: v_or_b32_e32 v21, v37, v39
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v51, v25, vcc
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[20:21]
-; SDAG-NEXT: v_ashrrev_i32_e32 v20, 31, v18
-; SDAG-NEXT: v_and_b32_e32 v55, v20, v34
-; SDAG-NEXT: v_mov_b32_e32 v53, 0
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v54
-; SDAG-NEXT: v_and_b32_e32 v54, v20, v27
-; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v22, v55
-; SDAG-NEXT: v_mov_b32_e32 v52, 0
-; SDAG-NEXT: v_or_b32_e32 v17, v53, v17
-; SDAG-NEXT: v_and_b32_e32 v53, v20, v12
-; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v54, vcc
-; SDAG-NEXT: v_or_b32_e32 v16, v52, v16
-; SDAG-NEXT: v_and_b32_e32 v18, 1, v20
-; SDAG-NEXT: v_and_b32_e32 v52, v20, v13
-; SDAG-NEXT: v_subb_u32_e32 v24, vcc, v24, v53, vcc
+; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v27, v52, vcc
; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v21, v19
-; SDAG-NEXT: v_mov_b32_e32 v20, v18
-; SDAG-NEXT: v_subb_u32_e32 v25, vcc, v25, v52, vcc
+; SDAG-NEXT: v_mov_b32_e32 v25, v19
+; SDAG-NEXT: v_mov_b32_e32 v24, v18
; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB4_9
; SDAG-NEXT: ; %bb.10: ; %Flow
; SDAG-NEXT: s_or_b64 exec, exec, s[10:11]
; SDAG-NEXT: .LBB4_11: ; %Flow11
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v15
; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v20
-; SDAG-NEXT: v_or_b32_e32 v20, v19, v15
-; SDAG-NEXT: v_or_b32_e32 v21, v18, v14
+; SDAG-NEXT: v_lshrrev_b32_e32 v20, 31, v17
+; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
+; SDAG-NEXT: v_or_b32_e32 v14, v14, v20
+; SDAG-NEXT: v_or_b32_e32 v20, v19, v17
+; SDAG-NEXT: v_or_b32_e32 v21, v18, v16
; SDAG-NEXT: .LBB4_12: ; %Flow12
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: v_mul_lo_u32 v9, v35, v9
-; SDAG-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v35, v8, 0
-; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v30, v35, 0
-; SDAG-NEXT: v_mul_lo_u32 v18, v33, v8
-; SDAG-NEXT: v_mov_b32_e32 v24, 0
-; SDAG-NEXT: v_add_i32_e32 v15, vcc, v15, v9
-; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v35, v[23:24]
-; SDAG-NEXT: v_add_i32_e32 v15, vcc, v15, v18
-; SDAG-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v10, v30, v[14:15]
+; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v35, v8, 0
+; SDAG-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v30, v35, 0
+; SDAG-NEXT: v_mov_b32_e32 v18, 0
; SDAG-NEXT: v_mul_lo_u32 v11, v11, v30
-; SDAG-NEXT: v_mov_b32_e32 v23, v8
-; SDAG-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v30, v33, v[23:24]
+; SDAG-NEXT: v_add_i32_e32 v19, vcc, v23, v9
+; SDAG-NEXT: v_mul_lo_u32 v23, v34, v8
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v35, v[17:18]
+; SDAG-NEXT: v_mul_lo_u32 v15, v15, v33
+; SDAG-NEXT: v_add_i32_e32 v23, vcc, v19, v23
+; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v10, v30, v[22:23]
+; SDAG-NEXT: v_mov_b32_e32 v17, v8
+; SDAG-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v30, v34, v[17:18]
; SDAG-NEXT: v_mul_lo_u32 v10, v10, v29
-; SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v15
-; SDAG-NEXT: v_add_i32_e32 v8, vcc, v9, v19
+; SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v23
+; SDAG-NEXT: v_add_i32_e32 v8, vcc, v9, v25
; SDAG-NEXT: v_addc_u32_e64 v9, s[4:5], 0, 0, vcc
-; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v33, v[8:9]
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v29, v34, v[8:9]
; SDAG-NEXT: v_add_i32_e32 v10, vcc, v10, v11
-; SDAG-NEXT: v_mul_lo_u32 v15, v16, v27
-; SDAG-NEXT: v_add_i32_e32 v8, vcc, v8, v14
+; SDAG-NEXT: v_add_i32_e32 v8, vcc, v8, v22
; SDAG-NEXT: v_addc_u32_e32 v9, vcc, v9, v10, vcc
-; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v22
-; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v18, vcc
+; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v16
+; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v24, vcc
; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v8, vcc
; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v9, vcc
; SDAG-NEXT: v_xor_b32_e32 v0, v0, v28
; SDAG-NEXT: v_mul_lo_u32 v10, v21, v13
; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v21, v12, 0
-; SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v34, v21, 0
-; SDAG-NEXT: v_xor_b32_e32 v1, v1, v31
+; SDAG-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v33, v21, 0
+; SDAG-NEXT: v_xor_b32_e32 v1, v1, v28
; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v28
; SDAG-NEXT: v_xor_b32_e32 v2, v2, v28
-; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v31, vcc
-; SDAG-NEXT: v_xor_b32_e32 v3, v3, v31
+; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v28, vcc
+; SDAG-NEXT: v_xor_b32_e32 v3, v3, v28
; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v28, vcc
; SDAG-NEXT: v_mul_lo_u32 v12, v20, v12
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v31, vcc
+; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v28, vcc
; SDAG-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v27, v21, v[23:24]
+; SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v32, v21, v[17:18]
; SDAG-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v16, v34, v[8:9]
-; SDAG-NEXT: v_mul_lo_u32 v14, v17, v34
-; SDAG-NEXT: v_mov_b32_e32 v23, v10
-; SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v34, v20, v[23:24]
-; SDAG-NEXT: v_add_i32_e32 v14, vcc, v14, v9
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v14, v33, v[8:9]
+; SDAG-NEXT: v_mov_b32_e32 v17, v10
+; SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v33, v20, v[17:18]
+; SDAG-NEXT: v_mul_lo_u32 v14, v14, v32
+; SDAG-NEXT: v_add_i32_e32 v15, vcc, v15, v9
; SDAG-NEXT: v_add_i32_e32 v9, vcc, v11, v13
; SDAG-NEXT: v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
-; SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v27, v20, v[9:10]
-; SDAG-NEXT: v_add_i32_e32 v11, vcc, v15, v14
+; SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v32, v20, v[9:10]
+; SDAG-NEXT: v_add_i32_e32 v11, vcc, v14, v15
; SDAG-NEXT: v_add_i32_e32 v8, vcc, v9, v8
; SDAG-NEXT: v_addc_u32_e32 v9, vcc, v10, v11, vcc
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v22
+; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v16
; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v12, vcc
; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v8, vcc
; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v9, vcc
-; SDAG-NEXT: v_xor_b32_e32 v4, v4, v26
-; SDAG-NEXT: v_xor_b32_e32 v5, v5, v32
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v26
-; SDAG-NEXT: v_xor_b32_e32 v6, v6, v26
-; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v32, vcc
-; SDAG-NEXT: v_xor_b32_e32 v7, v7, v32
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v26, vcc
-; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v32, vcc
+; SDAG-NEXT: v_xor_b32_e32 v4, v4, v31
+; SDAG-NEXT: v_xor_b32_e32 v5, v5, v31
+; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v31
+; SDAG-NEXT: v_xor_b32_e32 v6, v6, v31
+; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v31, vcc
+; SDAG-NEXT: v_xor_b32_e32 v7, v7, v31
+; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v6, v31, vcc
+; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v7, v31, vcc
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: v_srem_v2i128_vv:
@@ -2137,7 +2129,7 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_mov_b64 s[10:11], exec
; GISEL-NEXT: s_or_b64 s[6:7], s[4:5], s[6:7]
; GISEL-NEXT: s_mov_b64 s[12:13], 0
-; GISEL-NEXT: v_cndmask_b32_e64 v28, v10, 0, s[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v32, v10, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v18, v8, 0, s[4:5]
; GISEL-NEXT: v_cndmask_b32_e64 v19, v9, 0, s[4:5]
; GISEL-NEXT: s_xor_b64 s[8:9], s[6:7], s[10:11]
@@ -2254,21 +2246,21 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_lshl_b64 v[18:19], v[21:22], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v20, 31, v20
; GISEL-NEXT: v_or_b32_e32 v18, v18, v20
-; GISEL-NEXT: v_or_b32_e32 v28, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v32, v0, v2
; GISEL-NEXT: v_or_b32_e32 v33, v1, v3
; GISEL-NEXT: .LBB4_6: ; %Flow16
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_ashrrev_i32_e32 v32, 31, v7
-; GISEL-NEXT: v_xor_b32_e32 v1, v4, v32
-; GISEL-NEXT: v_xor_b32_e32 v2, v5, v32
-; GISEL-NEXT: v_xor_b32_e32 v3, v6, v32
-; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v1, v32
-; GISEL-NEXT: v_xor_b32_e32 v5, v7, v32
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v2, v32, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v28, 31, v7
+; GISEL-NEXT: v_xor_b32_e32 v1, v4, v28
+; GISEL-NEXT: v_xor_b32_e32 v2, v5, v28
+; GISEL-NEXT: v_xor_b32_e32 v3, v6, v28
+; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v1, v28
+; GISEL-NEXT: v_xor_b32_e32 v5, v7, v28
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v2, v28, vcc
; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v15
-; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v3, v32, vcc
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v3, v28, vcc
; GISEL-NEXT: v_xor_b32_e32 v12, v12, v0
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v32, vcc
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v28, vcc
; GISEL-NEXT: v_xor_b32_e32 v13, v13, v0
; GISEL-NEXT: v_sub_i32_e32 v35, vcc, v12, v0
; GISEL-NEXT: v_xor_b32_e32 v14, v14, v0
@@ -2449,25 +2441,25 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: .LBB4_12: ; %Flow12
; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v31, v18, 0
-; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v31, v28, 0
+; GISEL-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v31, v32, 0
; GISEL-NEXT: v_mul_lo_u32 v36, v30, v18
; GISEL-NEXT: v_mul_lo_u32 v3, v35, v3
; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v30, v33, v[0:1]
; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v35, v2, 0
; GISEL-NEXT: v_mul_lo_u32 v2, v34, v2
-; GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v28, v[14:15]
+; GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v32, v[14:15]
; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v35, v20, 0
; GISEL-NEXT: v_mad_u64_u32 v[26:27], vcc, v31, v33, v[23:24]
; GISEL-NEXT: v_mul_lo_u32 v23, v31, v19
; GISEL-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v34, v21, v[0:1]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v30, v28, v[26:27]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v30, v32, v[26:27]
; GISEL-NEXT: v_addc_u32_e64 v23, s[4:5], v25, v23, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v23, vcc, v23, v36, vcc
; GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v33, v[23:24]
; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v10, v22
; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v11, v0, vcc
; GISEL-NEXT: v_xor_b32_e32 v22, v10, v29
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v28, v[24:25]
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v17, v32, v[24:25]
; GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v12, v20, v[18:19]
; GISEL-NEXT: v_xor_b32_e32 v23, v0, v29
; GISEL-NEXT: v_subb_u32_e32 v0, vcc, v8, v1, vcc
@@ -2488,14 +2480,14 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v10, vcc
; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v11, vcc
; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v5, v15, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v32
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v32
-; GISEL-NEXT: v_xor_b32_e32 v8, v4, v32
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v6, v32
-; GISEL-NEXT: v_xor_b32_e32 v9, v5, v32
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v7, v32, vcc
-; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v8, v32, vcc
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v9, v32, vcc
+; GISEL-NEXT: v_xor_b32_e32 v6, v6, v28
+; GISEL-NEXT: v_xor_b32_e32 v7, v7, v28
+; GISEL-NEXT: v_xor_b32_e32 v8, v4, v28
+; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v6, v28
+; GISEL-NEXT: v_xor_b32_e32 v9, v5, v28
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v7, v28, vcc
+; GISEL-NEXT: v_subb_u32_e32 v6, vcc, v8, v28, vcc
+; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v9, v28, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = srem <2 x i128> %lhs, %rhs
ret <2 x i128> %shl
@@ -2777,8 +2769,8 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v24, v22, v24, vcc
; SDAG-NEXT: v_lshr_b64 v[22:23], v[6:7], v34
; SDAG-NEXT: v_cndmask_b32_e64 v26, v24, v4, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v29, 0, v23, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v28, 0, v22, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v31, 0, v23, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v30, 0, v22, vcc
; SDAG-NEXT: v_add_i32_e32 v38, vcc, -1, v12
; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v13, vcc
; SDAG-NEXT: v_addc_u32_e32 v48, vcc, -1, v14, vcc
@@ -2786,49 +2778,49 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
; SDAG-NEXT: s_mov_b64 s[10:11], 0
-; SDAG-NEXT: v_mov_b32_e32 v30, 0
-; SDAG-NEXT: v_mov_b32_e32 v31, 0
+; SDAG-NEXT: v_mov_b32_e32 v28, 0
+; SDAG-NEXT: v_mov_b32_e32 v29, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: .LBB5_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_lshl_b64 v[50:51], v[18:19], 1
; SDAG-NEXT: v_add_i32_e32 v34, vcc, -1, v34
-; SDAG-NEXT: v_or_b32_e32 v18, v30, v50
-; SDAG-NEXT: v_lshrrev_b32_e32 v30, 31, v27
+; SDAG-NEXT: v_or_b32_e32 v18, v28, v50
+; SDAG-NEXT: v_lshrrev_b32_e32 v28, 31, v27
; SDAG-NEXT: v_lshl_b64 v[26:27], v[26:27], 1
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v35, vcc
; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v19
-; SDAG-NEXT: v_or_b32_e32 v19, v31, v51
-; SDAG-NEXT: v_lshrrev_b32_e32 v31, 31, v21
+; SDAG-NEXT: v_or_b32_e32 v19, v29, v51
+; SDAG-NEXT: v_lshrrev_b32_e32 v29, 31, v21
; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v36, vcc
-; SDAG-NEXT: v_lshl_b64 v[28:29], v[28:29], 1
-; SDAG-NEXT: v_or_b32_e32 v26, v26, v31
+; SDAG-NEXT: v_lshl_b64 v[30:31], v[30:31], 1
+; SDAG-NEXT: v_or_b32_e32 v26, v26, v29
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v37, vcc
; SDAG-NEXT: v_or_b32_e32 v20, v20, v22
; SDAG-NEXT: v_sub_i32_e32 v22, vcc, v38, v26
-; SDAG-NEXT: v_or_b32_e32 v28, v28, v30
+; SDAG-NEXT: v_or_b32_e32 v30, v30, v28
; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v39, v27, vcc
-; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v48, v28, vcc
-; SDAG-NEXT: v_or_b32_e32 v30, v34, v36
-; SDAG-NEXT: v_or_b32_e32 v31, v35, v37
-; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v49, v29, vcc
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[30:31]
-; SDAG-NEXT: v_ashrrev_i32_e32 v30, 31, v22
-; SDAG-NEXT: v_and_b32_e32 v50, v30, v12
-; SDAG-NEXT: v_and_b32_e32 v31, v30, v13
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v48, v30, vcc
+; SDAG-NEXT: v_or_b32_e32 v28, v34, v36
+; SDAG-NEXT: v_or_b32_e32 v29, v35, v37
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v49, v31, vcc
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[28:29]
+; SDAG-NEXT: v_ashrrev_i32_e32 v28, 31, v22
+; SDAG-NEXT: v_and_b32_e32 v50, v28, v12
+; SDAG-NEXT: v_and_b32_e32 v29, v28, v13
; SDAG-NEXT: v_sub_i32_e32 v26, vcc, v26, v50
-; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v27, v31, vcc
-; SDAG-NEXT: v_and_b32_e32 v51, v30, v14
-; SDAG-NEXT: v_and_b32_e32 v22, 1, v30
-; SDAG-NEXT: v_and_b32_e32 v50, v30, v15
-; SDAG-NEXT: v_subb_u32_e32 v28, vcc, v28, v51, vcc
+; SDAG-NEXT: v_subb_u32_e32 v27, vcc, v27, v29, vcc
+; SDAG-NEXT: v_and_b32_e32 v51, v28, v14
+; SDAG-NEXT: v_and_b32_e32 v22, 1, v28
+; SDAG-NEXT: v_and_b32_e32 v50, v28, v15
+; SDAG-NEXT: v_subb_u32_e32 v30, vcc, v30, v51, vcc
; SDAG-NEXT: v_or_b32_e32 v21, v25, v21
; SDAG-NEXT: v_or_b32_e32 v20, v24, v20
; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v31, v23
-; SDAG-NEXT: v_mov_b32_e32 v30, v22
-; SDAG-NEXT: v_subb_u32_e32 v29, vcc, v29, v50, vcc
+; SDAG-NEXT: v_mov_b32_e32 v29, v23
+; SDAG-NEXT: v_mov_b32_e32 v28, v22
+; SDAG-NEXT: v_subb_u32_e32 v31, vcc, v31, v50, vcc
; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB5_9
; SDAG-NEXT: ; %bb.10: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 52def643e377e..a52d709ca28cc 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -10,26 +10,26 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-LABEL: v_srem_i128_vv:
; GFX9: ; %bb.0: ; %_udiv-special-cases
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i32_e32 v20, 31, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v20, vcc
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v20
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v20
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v20, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v18, 31, v3
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v18
+; GFX9-NEXT: v_xor_b32_e32 v1, v1, v18
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v18
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v18, vcc
+; GFX9-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX9-NEXT: v_xor_b32_e32 v3, v3, v18
+; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v18, vcc
; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v20, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v18, vcc
; GFX9-NEXT: v_xor_b32_e32 v4, v4, v8
; GFX9-NEXT: v_xor_b32_e32 v5, v5, v8
-; GFX9-NEXT: v_sub_co_u32_e32 v22, vcc, v4, v8
-; GFX9-NEXT: v_subb_co_u32_e32 v21, vcc, v5, v8, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v20, vcc, v4, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v19, vcc, v5, v8, vcc
; GFX9-NEXT: v_xor_b32_e32 v4, v6, v8
; GFX9-NEXT: v_xor_b32_e32 v5, v7, v8
; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v4, v8, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v5, v8, vcc
-; GFX9-NEXT: v_or_b32_e32 v7, v21, v5
-; GFX9-NEXT: v_or_b32_e32 v6, v22, v4
+; GFX9-NEXT: v_or_b32_e32 v7, v19, v5
+; GFX9-NEXT: v_or_b32_e32 v6, v20, v4
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_or_b32_e32 v7, v1, v3
; GFX9-NEXT: v_or_b32_e32 v6, v0, v2
@@ -38,9 +38,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_u32_e32 v6, 32, v6
; GFX9-NEXT: v_ffbh_u32_e32 v7, v5
; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX9-NEXT: v_ffbh_u32_e32 v7, v22
+; GFX9-NEXT: v_ffbh_u32_e32 v7, v20
; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
-; GFX9-NEXT: v_ffbh_u32_e32 v8, v21
+; GFX9-NEXT: v_ffbh_u32_e32 v8, v19
; GFX9-NEXT: v_min_u32_e32 v7, v7, v8
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7
@@ -88,10 +88,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-NEXT: v_add_co_u32_e32 v23, vcc, 1, v8
-; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v9, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v10, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v11, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v21, vcc, 1, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v22, vcc, 0, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v10, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v11, vcc
; GFX9-NEXT: v_sub_u32_e32 v11, 0x7f, v8
; GFX9-NEXT: v_sub_u32_e32 v9, 64, v11
; GFX9-NEXT: v_lshlrev_b64 v[6:7], v11, v[2:3]
@@ -116,71 +116,71 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_5
; GFX9-NEXT: ; %bb.2: ; %udiv-preheader
-; GFX9-NEXT: v_sub_u32_e32 v12, 64, v23
-; GFX9-NEXT: v_lshrrev_b64 v[10:11], v23, v[0:1]
+; GFX9-NEXT: v_sub_u32_e32 v12, 64, v21
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], v21, v[0:1]
; GFX9-NEXT: v_lshlrev_b64 v[12:13], v12, v[2:3]
-; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
+; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, 64, v21
; GFX9-NEXT: v_or_b32_e32 v12, v10, v12
-; GFX9-NEXT: v_subrev_u32_e32 v10, 64, v23
+; GFX9-NEXT: v_subrev_u32_e32 v10, 64, v21
; GFX9-NEXT: v_or_b32_e32 v13, v11, v13
; GFX9-NEXT: v_lshrrev_b64 v[10:11], v10, v[2:3]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v23
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v21
; GFX9-NEXT: v_cndmask_b32_e32 v11, v11, v13, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v11, v1, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v13, v11, v1, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v10, v12, vcc
-; GFX9-NEXT: v_lshrrev_b64 v[10:11], v23, v[2:3]
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v12, v0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v11, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v10, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v27, vcc, -1, v22
-; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v21, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v4, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-NEXT: v_lshrrev_b64 v[10:11], v21, v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v15, 0, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v14, 0, v10, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v25, vcc, -1, v20
+; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v19, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v4, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v5, vcc
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v18, 0
-; GFX9-NEXT: v_mov_b32_e32 v19, 0
+; GFX9-NEXT: v_mov_b32_e32 v16, 0
+; GFX9-NEXT: v_mov_b32_e32 v17, 0
; GFX9-NEXT: v_mov_b32_e32 v11, 0
; GFX9-NEXT: .LBB0_3: ; %udiv-do-while
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_lshlrev_b64 v[31:32], 1, v[8:9]
-; GFX9-NEXT: v_lshlrev_b64 v[16:17], 1, v[16:17]
-; GFX9-NEXT: v_or_b32_e32 v8, v18, v31
-; GFX9-NEXT: v_lshrrev_b32_e32 v18, 31, v15
+; GFX9-NEXT: v_lshlrev_b64 v[29:30], 1, v[8:9]
; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[14:15]
-; GFX9-NEXT: v_or_b32_e32 v16, v16, v18
-; GFX9-NEXT: v_lshrrev_b32_e32 v18, 31, v7
-; GFX9-NEXT: v_or_b32_e32 v14, v14, v18
-; GFX9-NEXT: v_sub_co_u32_e32 v18, vcc, v27, v14
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v28, v15, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v29, v16, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v18, vcc, v30, v17, vcc
-; GFX9-NEXT: v_ashrrev_i32_e32 v31, 31, v18
-; GFX9-NEXT: v_and_b32_e32 v18, v31, v22
+; GFX9-NEXT: v_or_b32_e32 v8, v16, v29
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 31, v13
+; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[12:13]
+; GFX9-NEXT: v_or_b32_e32 v14, v14, v16
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 31, v7
+; GFX9-NEXT: v_or_b32_e32 v12, v12, v16
+; GFX9-NEXT: v_sub_co_u32_e32 v16, vcc, v25, v12
+; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v26, v13, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v27, v14, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v28, v15, vcc
+; GFX9-NEXT: v_ashrrev_i32_e32 v29, 31, v16
+; GFX9-NEXT: v_and_b32_e32 v16, v29, v20
; GFX9-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; GFX9-NEXT: v_or_b32_e32 v9, v19, v32
-; GFX9-NEXT: v_and_b32_e32 v19, v31, v21
-; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18
-; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc
-; GFX9-NEXT: v_and_b32_e32 v18, v31, v4
-; GFX9-NEXT: v_and_b32_e32 v19, v31, v5
-; GFX9-NEXT: v_subb_co_u32_e32 v16, vcc, v16, v18, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v17, vcc, v17, v19, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v23, vcc, -1, v23
-; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v25, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, -1, v26, vcc
+; GFX9-NEXT: v_or_b32_e32 v9, v17, v30
+; GFX9-NEXT: v_and_b32_e32 v17, v29, v19
+; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, v12, v16
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v13, v17, vcc
+; GFX9-NEXT: v_and_b32_e32 v16, v29, v4
+; GFX9-NEXT: v_and_b32_e32 v17, v29, v5
+; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, v14, v16, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v17, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v21, vcc, -1, v21
+; GFX9-NEXT: v_addc_co_u32_e32 v22, vcc, -1, v22, vcc
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
-; GFX9-NEXT: v_or_b32_e32 v18, v23, v25
-; GFX9-NEXT: v_or_b32_e32 v19, v24, v26
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[18:19]
-; GFX9-NEXT: v_or3_b32 v6, v6, v10, v12
-; GFX9-NEXT: v_and_b32_e32 v10, 1, v31
-; GFX9-NEXT: v_or3_b32 v7, v7, 0, v13
+; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v23, vcc
+; GFX9-NEXT: v_mov_b32_e32 v16, 0
+; GFX9-NEXT: v_mov_b32_e32 v17, 0
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, -1, v24, vcc
+; GFX9-NEXT: v_or3_b32 v6, v6, v10, v16
+; GFX9-NEXT: v_or3_b32 v7, v7, 0, v17
+; GFX9-NEXT: v_or_b32_e32 v16, v21, v23
+; GFX9-NEXT: v_or_b32_e32 v17, v22, v24
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
+; GFX9-NEXT: v_and_b32_e32 v10, 1, v29
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v19, v11
-; GFX9-NEXT: v_mov_b32_e32 v18, v10
+; GFX9-NEXT: v_mov_b32_e32 v17, v11
+; GFX9-NEXT: v_mov_b32_e32 v16, v10
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB0_3
; GFX9-NEXT: ; %bb.4: ; %Flow
@@ -195,21 +195,21 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or_b32_e32 v13, v10, v8
; GFX9-NEXT: .LBB0_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v22, v13, 0
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v20, v13, 0
; GFX9-NEXT: v_mov_b32_e32 v10, 0
; GFX9-NEXT: v_mul_lo_u32 v11, v12, v4
; GFX9-NEXT: v_mul_lo_u32 v16, v13, v5
-; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v21, v13, v[9:10]
+; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v19, v13, v[9:10]
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v4, 0
-; GFX9-NEXT: v_mul_lo_u32 v13, v6, v21
-; GFX9-NEXT: v_mul_lo_u32 v17, v7, v22
+; GFX9-NEXT: v_mul_lo_u32 v13, v6, v19
+; GFX9-NEXT: v_mul_lo_u32 v17, v7, v20
; GFX9-NEXT: v_mov_b32_e32 v9, v14
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v22, v12, v[9:10]
+; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v20, v12, v[9:10]
; GFX9-NEXT: v_add3_u32 v5, v5, v16, v11
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v22, v[4:5]
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v20, v[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v15, v10
; GFX9-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, 0, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v21, v12, v[6:7]
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v19, v12, v[6:7]
; GFX9-NEXT: v_add3_u32 v5, v17, v5, v13
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v5, vcc
@@ -217,14 +217,14 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v9, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc
-; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20
-; GFX9-NEXT: v_xor_b32_e32 v1, v1, v20
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20
-; GFX9-NEXT: v_xor_b32_e32 v2, v2, v20
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v20, vcc
-; GFX9-NEXT: v_xor_b32_e32 v3, v3, v20
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v20, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v20, vcc
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v18
+; GFX9-NEXT: v_xor_b32_e32 v1, v1, v18
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v18
+; GFX9-NEXT: v_xor_b32_e32 v2, v2, v18
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v18, vcc
+; GFX9-NEXT: v_xor_b32_e32 v3, v3, v18
+; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v18, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v18, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-O0-LABEL: v_srem_i128_vv:
>From 5012d6c4314869cb66b60205ecd3132b0df38d9a Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Tue, 11 Aug 2026 01:50:14 -0500
Subject: [PATCH 8/9] Clean-up logic, update RUN line of a lit-test
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 64 ++++++++-----------
.../peephole-opt-beyond-cross-rc-copy.mir | 2 +-
2 files changed, 29 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 83099fbd1e2c6..b34f88e4ebe40 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1027,24 +1027,16 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// REG_SEQUENCE, etc.) and still fall back if the deeper trace dead-ends.
bool FoundSuitable = false;
RegSubRegPair SuitablePair = RegSubReg;
- // Fall back to the last suitable (sub)register source we remembered while
- // trying to reach a deeper one. Drop the speculative map edge leaving it so
- // getNewSource() terminates exactly at the fallback, and stop exploring any
- // remaining (PHI) work items.
- auto FallBackToSuitable = [&]() {
- CurSrcPair = SuitablePair;
- RewriteMap.erase(SuitablePair);
- SrcToLook.clear();
- };
+ // Set once a chain dead-ends (no deeper source is reachable). The decision of
+ // whether to fall back to the last suitable source or to give up is deferred
+ // to a single point after the loops below.
+ bool Aborted = false;
do {
CurSrcPair = SrcToLook.pop_back_val();
// As explained above, do not handle physical registers
if (CurSrcPair.Reg.isPhysical()) {
- if (FoundSuitable) {
- FallBackToSuitable();
- break;
- }
- return false;
+ Aborted = true;
+ break;
}
ValueTracker ValTracker(CurSrcPair.Reg, CurSrcPair.SubReg, *MRI, TII);
@@ -1055,13 +1047,8 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
ValueTrackerResult Res = ValTracker.getNextSource();
// Abort at the end of a chain (without finding a suitable source).
if (!Res.isValid()) {
- // If we already passed a suitable source while trying to reach a deeper
- // one, fall back to it instead of aborting.
- if (FoundSuitable) {
- FallBackToSuitable();
- break;
- }
- return false;
+ Aborted = true;
+ break;
}
// Insert the Def -> Use entry for the recently found source.
@@ -1076,11 +1063,7 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
if (CurSrcRes.getNumSources() > 1) {
LLVM_DEBUG(dbgs()
<< "findNextSource: found PHI cycle, aborting...\n");
- if (FoundSuitable) {
- FallBackToSuitable();
- break;
- }
- return false;
+ Aborted = true;
}
break;
}
@@ -1092,11 +1075,8 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
PHICount++;
if (PHICount >= RewritePHILimit) {
LLVM_DEBUG(dbgs() << "findNextSource: PHI limit reached\n");
- if (FoundSuitable) {
- FallBackToSuitable();
- break;
- }
- return false;
+ Aborted = true;
+ break;
}
for (unsigned i = 0; i < NumSrcs; ++i)
@@ -1110,11 +1090,8 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// the live-range of a physical register, unlike SSA virtual register,
// we will have to check that they aren't redefine before the related use.
if (CurSrcPair.Reg.isPhysical()) {
- if (FoundSuitable) {
- FallBackToSuitable();
- break;
- }
- return false;
+ Aborted = true;
+ break;
}
// Keep following the chain if the value isn't any better yet.
@@ -1140,8 +1117,23 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
// We found a suitable source, and are done with this chain.
break;
}
+
+ // A dead-ended chain ends all exploration; defer the fallback decision.
+ if (Aborted)
+ break;
} while (!SrcToLook.empty());
+ // Decide the rewrite point once: fall back to the last suitable (sub)register
+ // source we passed, or give up if there was none.
+ if (Aborted) {
+ if (!FoundSuitable)
+ return false;
+ // Drop the speculative map edge leaving the fallback so getNewSource()
+ // terminates exactly there.
+ CurSrcPair = SuitablePair;
+ RewriteMap.erase(SuitablePair);
+ }
+
// If we did not find a more suitable source, there is nothing to optimize.
return CurSrcPair.Reg != Reg;
}
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
index 5cc68825c2f55..17ec2837ab44c 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-beyond-cross-rc-copy.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx950 -run-pass peephole-opt -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.50 -run-pass peephole-opt -verify-machineinstrs -o - %s | FileCheck %s
# Reproduces the fp8_gemm pattern: an av_64_align2 COPY of a vreg_64_align2
# REG_SEQUENCE whose sub0 is used cross-block. PeepholeOptimizer::findNextSource
>From 939b8dc70a7958b624862d6c2f700d934d03fdb5 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Tue, 11 Aug 2026 02:21:05 -0500
Subject: [PATCH 9/9] Clarifications
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index b34f88e4ebe40..c5d942599fa42 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1123,13 +1123,11 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
break;
} while (!SrcToLook.empty());
- // Decide the rewrite point once: fall back to the last suitable (sub)register
- // source we passed, or give up if there was none.
if (Aborted) {
+ // If aborted with an invalid source, restore the suitable so far, if any.
if (!FoundSuitable)
return false;
- // Drop the speculative map edge leaving the fallback so getNewSource()
- // terminates exactly there.
+
CurSrcPair = SuitablePair;
RewriteMap.erase(SuitablePair);
}
More information about the llvm-commits
mailing list