[llvm] PeepholeOpt: Block subregister sources for REG_SEQUENCE copy coalescing (PR #194628)
Frederik Harwath via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 06:43:04 PDT 2026
https://github.com/frederik-h updated https://github.com/llvm/llvm-project/pull/194628
>From a0bd47c966a1d5c0b762cffff0e79c9de47954e8 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Fri, 24 Apr 2026 06:10:15 -0400
Subject: [PATCH 1/2] PeepholeOpt: Block subregister sources for REG_SEQUENCE
copy coalescing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Applying the coalescable copy optimization to REG_SEQUENCE instructions
can increase register pressure if findNextSource returns a source
with a subregister. Rewriting %A.sub0 to %B.sub1 will often
keep register %B alive in addition to %A.
Add an AcceptSubReg parameter to findNextSource to control when subregister
sources are allowed. Set it to false only for the REG_SEQUENCE case
in the coalescable copy optimization. In other places, e.g. the
uncoalescable copy optimizations, the subregister sources seem desirable.
Remove -new-reg-bank-select in the readlane.ll test since otherwise
GlobalISel creates illegal VGPR→SGPR copies. This looks like a
preexisting bug and peephole-opt should not be obliged to fix this.
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 26 +-
.../AMDGPU/GlobalISel/cvt_f32_ubyte.ll | 56 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 15 +-
.../CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll | 978 +--
.../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 396 +-
.../CodeGen/AMDGPU/GlobalISel/srem.i64.ll | 268 +-
.../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 116 +-
.../CodeGen/AMDGPU/a-v-flat-atomic-cmpxchg.ll | 128 +-
llvm/test/CodeGen/AMDGPU/add_i1.ll | 13 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 7201 +++++++++--------
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 200 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll | 304 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 736 +-
.../atomic_optimizations_global_pointer.ll | 2608 +++---
llvm/test/CodeGen/AMDGPU/bf16.ll | 95 +-
.../AMDGPU/buffer-fat-pointers-memcpy.ll | 248 +-
.../CodeGen/AMDGPU/call-argument-types.ll | 63 +-
.../CodeGen/AMDGPU/calling-conventions.ll | 227 +-
llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll | 72 +-
llvm/test/CodeGen/AMDGPU/ctpop64.ll | 36 +-
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 20 +-
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 126 +-
llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll | 13 +-
llvm/test/CodeGen/AMDGPU/fceil64.ll | 24 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 58 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 58 +-
.../AMDGPU/flat_atomics_i64_noprivate.ll | 132 +-
.../CodeGen/AMDGPU/flat_atomics_i64_system.ll | 412 +-
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 40 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 166 +-
llvm/test/CodeGen/AMDGPU/fptrunc.ll | 82 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 115 +-
.../AMDGPU/gfx-callable-return-types.ll | 8 +-
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 76 +-
.../identical-subrange-spill-infloop.ll | 163 +-
llvm/test/CodeGen/AMDGPU/idot2.ll | 52 +-
llvm/test/CodeGen/AMDGPU/idot4s.ll | 124 +-
llvm/test/CodeGen/AMDGPU/idot4u.ll | 310 +-
llvm/test/CodeGen/AMDGPU/idot8u.ll | 36 +-
.../CodeGen/AMDGPU/indirect-addressing-si.ll | 84 +-
llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll | 298 +-
llvm/test/CodeGen/AMDGPU/kernel-args.ll | 9 +-
...llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll | 16 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll | 380 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 168 +-
.../CodeGen/AMDGPU/llvm.amdgcn.readlane.ll | 3 +-
.../AMDGPU/llvm.amdgcn.sched.group.barrier.ll | 340 +-
.../AMDGPU/llvm.amdgcn.smfmac.gfx950.ll | 320 +-
llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll | 105 +-
llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll | 6 +-
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 128 +-
llvm/test/CodeGen/AMDGPU/load-constant-i32.ll | 36 +-
llvm/test/CodeGen/AMDGPU/load-global-f32.ll | 100 +-
llvm/test/CodeGen/AMDGPU/load-global-i32.ll | 205 +-
llvm/test/CodeGen/AMDGPU/mad-combine.ll | 126 +-
llvm/test/CodeGen/AMDGPU/memmove-var-size.ll | 336 +-
.../AMDGPU/memset-param-combinations.ll | 811 +-
llvm/test/CodeGen/AMDGPU/memset-pattern.ll | 58 +-
llvm/test/CodeGen/AMDGPU/merge-stores.ll | 22 +-
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll | 644 +-
.../AMDGPU/move-to-valu-atomicrmw-system.ll | 19 +-
llvm/test/CodeGen/AMDGPU/mul.ll | 50 +-
llvm/test/CodeGen/AMDGPU/mul_int24.ll | 31 +-
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 240 +-
.../peephole-opt-reg-sequence-copy-folding.ll | 275 +
.../AMDGPU/peephole-opt-regseq-removal.mir | 6 +-
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 274 +-
.../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll | 188 +-
llvm/test/CodeGen/AMDGPU/select.f16.ll | 122 +-
llvm/test/CodeGen/AMDGPU/shl.ll | 22 +-
.../AMDGPU/shufflevector.v2i64.v8i64.ll | 468 +-
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 320 +-
llvm/test/CodeGen/AMDGPU/spill-vgpr.ll | 16 +-
llvm/test/CodeGen/AMDGPU/sra.ll | 22 +-
llvm/test/CodeGen/AMDGPU/srl.ll | 22 +-
llvm/test/CodeGen/AMDGPU/sub_i1.ll | 13 +-
llvm/test/CodeGen/AMDGPU/udiv.ll | 14 +-
llvm/test/CodeGen/AMDGPU/uint_to_fp.i64.ll | 42 +-
.../unspill-vgpr-after-rewrite-vgpr-mfma.ll | 560 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 10 +-
.../AMDGPU/vgpr-mark-last-scratch-load.ll | 383 +-
.../test/CodeGen/AMDGPU/vni8-across-blocks.ll | 108 +-
llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll | 149 +-
.../CodeGen/AMDGPU/widen-vselect-and-mask.ll | 26 +-
84 files changed, 11826 insertions(+), 11550 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/peephole-opt-reg-sequence-copy-folding.ll
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 9365ea883eec9..2a5ac2de9e3bb 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -451,13 +451,15 @@ class PeepholeOptimizer : private MachineFunction::Delegate {
SmallPtrSetImpl<MachineInstr *> &LocalMIs);
bool optimizeCondBranch(MachineInstr &MI);
- bool optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter);
+ bool optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter,
+ bool AcceptSubReg = true);
bool optimizeCoalescableCopy(MachineInstr &MI);
bool optimizeUncoalescableCopy(MachineInstr &MI,
SmallPtrSetImpl<MachineInstr *> &LocalMIs);
bool optimizeRecurrence(MachineInstr &PHI);
bool findNextSource(const TargetRegisterClass *DefRC, unsigned DefSubReg,
- RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap);
+ RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap,
+ bool AcceptSubReg = true);
bool isMoveImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
DenseMap<Register, MachineInstr *> &ImmDefMIs);
bool foldImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
@@ -989,7 +991,8 @@ bool PeepholeOptimizer::optimizeCondBranch(MachineInstr &MI) {
bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
unsigned DefSubReg,
RegSubRegPair RegSubReg,
- RewriteMapTy &RewriteMap) {
+ RewriteMapTy &RewriteMap,
+ bool AcceptSubReg) {
// Do not try to find a new source for a physical register.
// So far we do not have any motivating example for doing that.
// Thus, instead of maintaining untested code, we will revisit that if
@@ -1066,6 +1069,9 @@ bool PeepholeOptimizer::findNextSource(const TargetRegisterClass *DefRC,
if (PHICount > 0 && CurSrcPair.SubReg != 0)
continue;
+ if (!AcceptSubReg && CurSrcPair.SubReg != 0)
+ continue;
+
// We found a suitable source, and are done with this chain.
break;
}
@@ -1161,7 +1167,8 @@ getNewSource(MachineRegisterInfo *MRI, const TargetInstrInfo *TII,
return RegSubRegPair(0, 0);
}
-bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter) {
+bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter,
+ bool AcceptSubReg) {
bool Changed = false;
// Get the right rewriter for the current copy.
// Rewrite each rewritable source.
@@ -1182,7 +1189,8 @@ bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter) {
RewriteMapTy RewriteMap;
// Try to find a more suitable source. If we failed to do so, or get the
// actual source, move to the next source.
- if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap))
+ if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap,
+ AcceptSubReg))
continue;
// Get the new source to rewrite. TODO: Only enable handling of multiple
@@ -1250,8 +1258,12 @@ bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr &MI) {
return optimizeCoalescableCopyImpl(InsertSubregRewriter(MI));
case TargetOpcode::EXTRACT_SUBREG:
return optimizeCoalescableCopyImpl(ExtractSubregRewriter(MI, *TII));
- case TargetOpcode::REG_SEQUENCE:
- return optimizeCoalescableCopyImpl(RegSequenceRewriter(MI));
+ case TargetOpcode::REG_SEQUENCE: {
+ // Subreg sources can increase register pressure by keeping
+ // multiple wide registers alive.
+ bool AcceptSubReg = false;
+ return optimizeCoalescableCopyImpl(RegSequenceRewriter(MI), AcceptSubReg);
+ }
default:
// Handle uncoalescable copy-like instructions.
if (MI.isBitcast() || MI.isRegSequenceLike() || MI.isInsertSubregLike() ||
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index de586a5921724..cc02ba8864323 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -1434,34 +1434,32 @@ define float @v_test_sitofp_i64_byte_to_f32(i64 %arg0) {
; SI-LABEL: v_test_sitofp_i64_byte_to_f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, 0xff
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v0
-; SI-NEXT: v_ffbh_i32_e32 v0, 0
-; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v0
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_min_u32_e32 v3, 32, v0
-; SI-NEXT: v_lshl_b64 v[0:1], v[1:2], v3
+; SI-NEXT: v_ffbh_i32_e32 v2, 0
+; SI-NEXT: v_add_i32_e32 v2, vcc, -1, v2
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_min_u32_e32 v2, 32, v2
+; SI-NEXT: v_lshl_b64 v[0:1], v[0:1], v2
; SI-NEXT: v_min_u32_e32 v0, 1, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_cvt_f32_i32_e32 v0, v0
-; SI-NEXT: v_sub_i32_e32 v1, vcc, 32, v3
+; SI-NEXT: v_sub_i32_e32 v1, vcc, 32, v2
; SI-NEXT: v_ldexp_f32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_test_sitofp_i64_byte_to_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, 0xff
-; VI-NEXT: v_and_b32_e32 v1, 0xff, v0
-; VI-NEXT: v_ffbh_i32_e32 v0, 0
-; VI-NEXT: v_add_u32_e32 v0, vcc, -1, v0
-; VI-NEXT: v_mov_b32_e32 v2, 0
-; VI-NEXT: v_min_u32_e32 v3, 32, v0
-; VI-NEXT: v_lshlrev_b64 v[0:1], v3, v[1:2]
+; VI-NEXT: v_ffbh_i32_e32 v2, 0
+; VI-NEXT: v_add_u32_e32 v2, vcc, -1, v2
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; VI-NEXT: v_min_u32_e32 v2, 32, v2
+; VI-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; VI-NEXT: v_min_u32_e32 v0, 1, v0
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_cvt_f32_i32_e32 v0, v0
-; VI-NEXT: v_sub_u32_e32 v1, vcc, 32, v3
+; VI-NEXT: v_sub_u32_e32 v1, vcc, 32, v2
; VI-NEXT: v_ldexp_f32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
%masked = and i64 %arg0, 255
@@ -1473,32 +1471,30 @@ define float @v_test_uitofp_i64_byte_to_f32(i64 %arg0) {
; SI-LABEL: v_test_uitofp_i64_byte_to_f32:
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, 0xff
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v0
-; SI-NEXT: v_ffbh_u32_e32 v0, 0
-; SI-NEXT: v_mov_b32_e32 v2, 0
-; SI-NEXT: v_min_u32_e32 v3, 32, v0
-; SI-NEXT: v_lshl_b64 v[0:1], v[1:2], v3
+; SI-NEXT: v_ffbh_u32_e32 v2, 0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT: v_min_u32_e32 v2, 32, v2
+; SI-NEXT: v_lshl_b64 v[0:1], v[0:1], v2
; SI-NEXT: v_min_u32_e32 v0, 1, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
; SI-NEXT: v_cvt_f32_u32_e32 v0, v0
-; SI-NEXT: v_sub_i32_e32 v1, vcc, 32, v3
+; SI-NEXT: v_sub_i32_e32 v1, vcc, 32, v2
; SI-NEXT: v_ldexp_f32_e32 v0, v0, v1
; SI-NEXT: s_setpc_b64 s[30:31]
;
; VI-LABEL: v_test_uitofp_i64_byte_to_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, 0xff
-; VI-NEXT: v_and_b32_e32 v1, 0xff, v0
-; VI-NEXT: v_ffbh_u32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v2, 0
-; VI-NEXT: v_min_u32_e32 v3, 32, v0
-; VI-NEXT: v_lshlrev_b64 v[0:1], v3, v[1:2]
+; VI-NEXT: v_ffbh_u32_e32 v2, 0
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; VI-NEXT: v_min_u32_e32 v2, 32, v2
+; VI-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
; VI-NEXT: v_min_u32_e32 v0, 1, v0
; VI-NEXT: v_or_b32_e32 v0, v1, v0
; VI-NEXT: v_cvt_f32_u32_e32 v0, v0
-; VI-NEXT: v_sub_u32_e32 v1, vcc, 32, v3
+; VI-NEXT: v_sub_u32_e32 v1, vcc, 32, v2
; VI-NEXT: v_ldexp_f32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
%masked = and i64 %arg0, 255
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index fc4ff6f835963..d47b37ece842e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -1637,7 +1637,6 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX6-LABEL: v_lshr_i65:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, 1
; GFX6-NEXT: v_mov_b32_e32 v5, 0
; GFX6-NEXT: v_and_b32_e32 v4, 1, v2
; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 64, v3
@@ -1660,7 +1659,6 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX8-LABEL: v_lshr_i65:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, 1
; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: v_and_b32_e32 v4, 1, v2
; GFX8-NEXT: v_sub_u32_e32 v8, vcc, 64, v3
@@ -1683,7 +1681,6 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX9-LABEL: v_lshr_i65:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, 1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_and_b32_e32 v4, 1, v2
; GFX9-NEXT: v_sub_u32_e32 v8, 64, v3
@@ -1706,7 +1703,6 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX10-LABEL: v_lshr_i65:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, 1
; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: v_and_b32_e32 v4, 1, v2
; GFX10-NEXT: v_sub_nc_u32_e32 v2, 64, v3
@@ -1729,8 +1725,7 @@ define i65 @v_lshr_i65(i65 %value, i65 %amount) {
; GFX11-LABEL: v_lshr_i65:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 0
-; GFX11-NEXT: v_and_b32_e32 v4, 1, v2
+; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v4, 1, v2
; GFX11-NEXT: v_sub_nc_u32_e32 v2, 64, v3
; GFX11-NEXT: v_lshrrev_b64 v[6:7], v3, v[0:1]
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 64, v3
@@ -1755,7 +1750,6 @@ define i65 @v_lshr_i65_33(i65 %value) {
; GFX6-LABEL: v_lshr_i65_33:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, 1
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v1, 0
; GFX6-NEXT: v_and_b32_e32 v0, 1, v2
@@ -1768,7 +1762,6 @@ define i65 @v_lshr_i65_33(i65 %value) {
; GFX8-LABEL: v_lshr_i65_33:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, 1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_and_b32_e32 v0, 1, v2
@@ -1781,7 +1774,6 @@ define i65 @v_lshr_i65_33(i65 %value) {
; GFX9-LABEL: v_lshr_i65_33:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, 1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_and_b32_e32 v0, 1, v2
@@ -1794,7 +1786,6 @@ define i65 @v_lshr_i65_33(i65 %value) {
; GFX10-LABEL: v_lshr_i65_33:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, 1
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_and_b32_e32 v0, 1, v2
@@ -1807,8 +1798,8 @@ define i65 @v_lshr_i65_33(i65 %value) {
; GFX11-LABEL: v_lshr_i65_33:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, 1
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 1, v2
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 1, v2
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: v_lshrrev_b32_e32 v2, 1, v3
; GFX11-NEXT: v_lshlrev_b64 v[0:1], 31, v[0:1]
; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index d02ff99cca3eb..2705afc3eb262 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -420,187 +420,187 @@ define <2 x i64> @v_sdiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_ashrrev_i32_e32 v5, 31, v1
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v9, v[12:13]
+; GISEL-NEXT: v_xor_b32_e32 v17, v0, v5
+; GISEL-NEXT: v_mul_lo_u32 v0, v9, v11
; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v1, v5, vcc
-; GISEL-NEXT: v_xor_b32_e32 v17, v1, v5
; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v18, v19, v[13:14]
-; GISEL-NEXT: v_xor_b32_e32 v16, v0, v5
-; GISEL-NEXT: v_mul_lo_u32 v0, v9, v11
-; GISEL-NEXT: v_mul_hi_u32 v1, v19, v11
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v15
+; GISEL-NEXT: v_mul_hi_u32 v13, v19, v11
; GISEL-NEXT: v_mul_hi_u32 v11, v9, v11
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v5
+; GISEL-NEXT: v_mul_lo_u32 v12, v19, v15
+; GISEL-NEXT: v_xor_b32_e32 v8, v5, v8
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v1, v9, v15
+; GISEL-NEXT: v_mul_lo_u32 v13, v9, v15
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
; GISEL-NEXT: v_mul_hi_u32 v12, v19, v15
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v9, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v9, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v9, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v11, v16, v1
-; GISEL-NEXT: v_mul_hi_u32 v12, v16, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_mul_hi_u32 v13, v9, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v12, v17, v1
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT: v_mul_hi_u32 v11, v16, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v19, v0
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v9, v11, vcc
+; GISEL-NEXT: v_mul_lo_u32 v11, v1, v0
+; GISEL-NEXT: v_mul_lo_u32 v12, v17, v9
+; GISEL-NEXT: v_mul_hi_u32 v13, v17, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v1, v0
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v13, v1, v9
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v0, v9
-; GISEL-NEXT: v_mul_hi_u32 v12, v17, v1
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v15, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v11, v9
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v12, v9
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v18, v[1:2]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
+; GISEL-NEXT: v_mul_hi_u32 v12, v17, v9
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v12
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v0, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v9, v13
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v10, v18, v[12:13]
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v17, v11
; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v7
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v4, v15, v[11:12]
-; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], v17, v13, vcc
-; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v17, v13
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v4, v0, v[13:14]
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v1, v15, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v15
; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v9
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v7, v9, s[4:5]
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v7, v9, s[4:5]
; GISEL-NEXT: v_xor_b32_e32 v7, v6, v9
-; GISEL-NEXT: v_xor_b32_e32 v6, v11, v9
-; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v6
+; GISEL-NEXT: v_xor_b32_e32 v6, v12, v9
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v7
+; GISEL-NEXT: v_cvt_f32_u32_e32 v13, v6
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v10
-; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v10
-; GISEL-NEXT: v_subbrev_u32_e32 v16, vcc, 0, v1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v10
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v10
+; GISEL-NEXT: v_mac_f32_e32 v12, 0x4f800000, v13
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v12, v12
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v11, v10
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v10
+; GISEL-NEXT: v_mul_f32_e32 v10, 0x5f7ffffc, v12
+; GISEL-NEXT: v_mul_f32_e32 v11, 0x2f800000, v10
+; GISEL-NEXT: v_trunc_f32_e32 v11, v11
+; GISEL-NEXT: v_mac_f32_e32 v10, 0xcf800000, v11
+; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v10
; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v7
-; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v11
; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v6, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v17, 0
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v17, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v1, v4
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v20, v19, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v11, v19, v10
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[6:7], v21, v17, v[12:13]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v4
+; GISEL-NEXT: v_mul_lo_u32 v12, v17, v14
+; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT: v_mul_hi_u32 v12, v17, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v10, v19, v10
+; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v4
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v19, v[1:2]
-; GISEL-NEXT: v_mul_lo_u32 v1, v19, v0
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v21, v17, v[10:11]
-; GISEL-NEXT: v_mul_lo_u32 v10, v17, v12
-; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v17, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
-; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
-; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v10
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v0
; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v16, v4
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v14, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v14, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, 1, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v4, v13, v16, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, 0, v18, vcc
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, 1, v10
-; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v13, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
-; GISEL-NEXT: v_mul_lo_u32 v11, v19, v12
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GISEL-NEXT: v_mul_hi_u32 v11, v17, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v12, v19, v12
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v11
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v15, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, 1, v11
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v12, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v11, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v13, v17, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v14, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v11
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v11, v1
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v17, v0
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v12, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v15, v10, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v17, v10
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v11, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v13, 0
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v18, v12, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v0, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v14, v[11:12]
+; GISEL-NEXT: v_xor_b32_e32 v11, v4, v8
; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v14, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v5, v8
-; GISEL-NEXT: v_xor_b32_e32 v8, v4, v1
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v18, v13, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v12, v[10:11]
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
-; GISEL-NEXT: v_xor_b32_e32 v16, v2, v15
-; GISEL-NEXT: v_mul_lo_u32 v2, v14, v0
-; GISEL-NEXT: v_mul_lo_u32 v5, v12, v4
-; GISEL-NEXT: v_xor_b32_e32 v17, v3, v15
-; GISEL-NEXT: v_mul_hi_u32 v3, v12, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v14, v0
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v13, v[0:1]
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v15
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v3, v15, vcc
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v15
+; GISEL-NEXT: v_mul_lo_u32 v0, v14, v10
+; GISEL-NEXT: v_mul_lo_u32 v2, v13, v4
+; GISEL-NEXT: v_xor_b32_e32 v17, v1, v15
+; GISEL-NEXT: v_mul_hi_u32 v1, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v3, v14, v4
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
-; GISEL-NEXT: v_mul_hi_u32 v5, v12, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v1, v14, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_mul_hi_u32 v2, v13, v4
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v3
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_mul_hi_u32 v4, v14, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v14, v2, vcc
-; GISEL-NEXT: v_mul_lo_u32 v3, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v4, v16, v2
-; GISEL-NEXT: v_mul_hi_u32 v5, v16, v0
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v14, v1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v2, v17, v0
+; GISEL-NEXT: v_mul_lo_u32 v3, v16, v1
+; GISEL-NEXT: v_mul_hi_u32 v4, v16, v0
; GISEL-NEXT: v_mul_hi_u32 v0, v17, v0
-; GISEL-NEXT: v_xor_b32_e32 v10, v13, v1
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GISEL-NEXT: v_xor_b32_e32 v10, v12, v8
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, v17, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; GISEL-NEXT: v_mul_hi_u32 v4, v16, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v5, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v4, v17, v1
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_mul_hi_u32 v3, v16, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v5, v17, v2
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v3
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v0, v2
+; GISEL-NEXT: v_mul_hi_u32 v1, v17, v1
; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v7, v12, 0
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v5, v0
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v0
; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v7, v13, v[3:4]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v8, v1
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v10, v1, vcc
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v8
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v10, v8, vcc
; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v6, v12, v[4:5]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v16, v2
; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v17, v10, vcc
@@ -1824,267 +1824,267 @@ define <2 x i64> @v_sdiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
; GISEL-LABEL: v_sdiv_v2i64_pow2_shl_denom:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v11, 0x1000
-; GISEL-NEXT: v_mov_b32_e32 v12, 0
-; GISEL-NEXT: v_lshl_b64 v[7:8], v[11:12], v4
-; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v8
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v4
-; GISEL-NEXT: v_addc_u32_e32 v7, vcc, v8, v4, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, v5, v4
-; GISEL-NEXT: v_xor_b32_e32 v5, v7, v4
-; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v8
-; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v5
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 0, v8
+; GISEL-NEXT: v_mov_b32_e32 v7, 0x1000
+; GISEL-NEXT: v_mov_b32_e32 v8, 0
+; GISEL-NEXT: v_lshl_b64 v[9:10], v[7:8], v4
+; GISEL-NEXT: v_lshl_b64 v[7:8], v[7:8], v6
+; GISEL-NEXT: v_ashrrev_i32_e32 v4, 31, v10
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v9, v4
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, v10, v4, vcc
+; GISEL-NEXT: v_xor_b32_e32 v10, v5, v4
+; GISEL-NEXT: v_xor_b32_e32 v5, v9, v4
+; GISEL-NEXT: v_cvt_f32_u32_e32 v9, v10
+; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v5
+; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 0, v10
; GISEL-NEXT: v_subb_u32_e32 v17, vcc, 0, v5, vcc
-; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v9
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v7, v7
-; GISEL-NEXT: v_mul_f32_e32 v7, 0x5f7ffffc, v7
-; GISEL-NEXT: v_mul_f32_e32 v9, 0x2f800000, v7
-; GISEL-NEXT: v_trunc_f32_e32 v9, v9
-; GISEL-NEXT: v_mac_f32_e32 v7, 0xcf800000, v9
-; GISEL-NEXT: v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v9
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v7, 0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v16, v15, v[10:11]
-; GISEL-NEXT: v_mul_lo_u32 v18, v15, v9
-; GISEL-NEXT: v_mul_hi_u32 v19, v7, v9
-; GISEL-NEXT: v_mul_hi_u32 v20, v15, v9
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v7, v[13:14]
-; GISEL-NEXT: v_mul_lo_u32 v10, v7, v9
-; GISEL-NEXT: v_mul_lo_u32 v14, v15, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v18, v10
+; GISEL-NEXT: v_mac_f32_e32 v9, 0x4f800000, v11
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v9, v9
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v8
+; GISEL-NEXT: v_mul_f32_e32 v9, 0x5f7ffffc, v9
+; GISEL-NEXT: v_mul_f32_e32 v11, 0x2f800000, v9
+; GISEL-NEXT: v_trunc_f32_e32 v11, v11
+; GISEL-NEXT: v_mac_f32_e32 v9, 0xcf800000, v11
+; GISEL-NEXT: v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT: v_cvt_u32_f32_e32 v15, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v16, v15, v[12:13]
+; GISEL-NEXT: v_mul_lo_u32 v18, v15, v11
+; GISEL-NEXT: v_mul_hi_u32 v19, v9, v11
+; GISEL-NEXT: v_mul_hi_u32 v20, v15, v11
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v9, v[13:14]
+; GISEL-NEXT: v_mul_lo_u32 v12, v9, v11
+; GISEL-NEXT: v_mul_lo_u32 v14, v15, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v18, v12
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v19
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
-; GISEL-NEXT: v_mul_hi_u32 v13, v7, v9
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v19
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_mul_hi_u32 v13, v9, v11
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v14, v20
; GISEL-NEXT: v_cndmask_b32_e64 v18, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v14, vcc, v18, v14
-; GISEL-NEXT: v_mul_hi_u32 v9, v15, v9
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v11, v15, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v7, v10
-; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v15, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v18, 0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v16, v19, v[10:11]
-; GISEL-NEXT: v_mul_lo_u32 v7, v19, v9
-; GISEL-NEXT: v_lshl_b64 v[11:12], v[11:12], v6
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v9, v12
+; GISEL-NEXT: v_addc_u32_e32 v19, vcc, v15, v11, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v16, v18, 0
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v16, v19, v[12:13]
+; GISEL-NEXT: v_mul_lo_u32 v9, v19, v11
; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v17, v18, v[13:14]
-; GISEL-NEXT: v_mul_hi_u32 v13, v18, v9
-; GISEL-NEXT: v_ashrrev_i32_e32 v6, 31, v12
-; GISEL-NEXT: v_mul_lo_u32 v10, v18, v15
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v13
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v1
-; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v7
-; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], v1, v7, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v16, v0, v7
-; GISEL-NEXT: v_xor_b32_e32 v17, v1, v7
+; GISEL-NEXT: v_mul_hi_u32 v13, v18, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v19, v11
+; GISEL-NEXT: v_mul_lo_u32 v12, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v1
+; GISEL-NEXT: v_add_i32_e64 v0, s[4:5], v0, v9
+; GISEL-NEXT: v_xor_b32_e32 v17, v0, v9
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v1, v19, v9
-; GISEL-NEXT: v_mul_lo_u32 v9, v19, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_mul_hi_u32 v10, v18, v15
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_mul_hi_u32 v10, v19, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v9, v1
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v10, v1
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
+; GISEL-NEXT: v_mul_hi_u32 v12, v18, v15
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_mul_hi_u32 v13, v19, v15
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_addc_u32_e64 v1, s[4:5], v1, v9, s[4:5]
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v18, v0
-; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v19, v1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v9, v17, v0
-; GISEL-NEXT: v_mul_lo_u32 v10, v16, v1
-; GISEL-NEXT: v_mul_hi_u32 v13, v16, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v17, v0
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v13, v17, v1
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_mul_hi_u32 v10, v16, v1
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_xor_b32_e32 v1, v1, v9
+; GISEL-NEXT: v_addc_u32_e32 v11, vcc, v19, v11, vcc
+; GISEL-NEXT: v_mul_lo_u32 v12, v1, v0
+; GISEL-NEXT: v_mul_lo_u32 v13, v17, v11
+; GISEL-NEXT: v_mul_hi_u32 v14, v17, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v1, v0
+; GISEL-NEXT: v_mul_hi_u32 v15, v1, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v13
; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, v0, v9
-; GISEL-NEXT: v_mul_hi_u32 v13, v17, v1
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v15, 0
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, v13, v9
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v8, v18, v[1:2]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v16, v0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v5, v15, v[9:10]
-; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v17, v13, vcc
-; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v17, v13
-; GISEL-NEXT: v_add_i32_e64 v9, s[4:5], v11, v6
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v12, v6, s[4:5]
-; GISEL-NEXT: v_xor_b32_e32 v10, v9, v6
-; GISEL-NEXT: v_xor_b32_e32 v9, v11, v6
-; GISEL-NEXT: v_cvt_f32_u32_e32 v11, v10
-; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v9
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v14, v1, v11
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT: v_mul_hi_u32 v13, v17, v11
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v14, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v14, v13
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v10, v0, 0
+; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT: v_add_i32_e32 v18, vcc, v15, v13
+; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v10, v18, v[12:13]
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v17, v11
+; GISEL-NEXT: v_xor_b32_e32 v9, v9, v4
+; GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v5, v0, v[13:14]
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], v1, v15, vcc
+; GISEL-NEXT: v_sub_i32_e64 v1, s[4:5], v1, v15
+; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v7, v6
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v8, v6, s[4:5]
+; GISEL-NEXT: v_xor_b32_e32 v8, v7, v6
+; GISEL-NEXT: v_xor_b32_e32 v7, v12, v6
+; GISEL-NEXT: v_cvt_f32_u32_e32 v12, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v13, v7
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v8
-; GISEL-NEXT: v_mac_f32_e32 v11, 0x4f800000, v12
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v11, v11
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v8
-; GISEL-NEXT: v_subbrev_u32_e32 v17, vcc, 0, v1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v0, v8
-; GISEL-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v11
-; GISEL-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GISEL-NEXT: v_trunc_f32_e32 v1, v1
-; GISEL-NEXT: v_mac_f32_e32 v0, 0xcf800000, v1
-; GISEL-NEXT: v_cvt_u32_f32_e32 v8, v0
-; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v10
-; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v1
-; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v9, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v20, v8, 0
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v17, v5
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v20, v19, v[1:2]
-; GISEL-NEXT: v_mul_lo_u32 v1, v19, v0
-; GISEL-NEXT: v_mad_u64_u32 v[13:14], s[6:7], v21, v8, v[11:12]
-; GISEL-NEXT: v_mul_lo_u32 v11, v8, v13
-; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v11
-; GISEL-NEXT: v_mul_hi_u32 v11, v8, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[6:7]
-; GISEL-NEXT: v_mul_hi_u32 v0, v19, v0
-; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], v1, v11
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v17, v5
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v11, v10
+; GISEL-NEXT: v_mac_f32_e32 v12, 0x4f800000, v13
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v12, v12
+; GISEL-NEXT: v_sub_i32_e32 v11, vcc, v11, v10
+; GISEL-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v11, v10
+; GISEL-NEXT: v_mul_f32_e32 v10, 0x5f7ffffc, v12
+; GISEL-NEXT: v_mul_f32_e32 v11, 0x2f800000, v10
+; GISEL-NEXT: v_trunc_f32_e32 v11, v11
+; GISEL-NEXT: v_mac_f32_e32 v10, 0xcf800000, v11
+; GISEL-NEXT: v_cvt_u32_f32_e32 v17, v10
+; GISEL-NEXT: v_sub_i32_e64 v20, s[6:7], 0, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v19, v11
+; GISEL-NEXT: v_subb_u32_e64 v21, s[6:7], 0, v7, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v20, v17, 0
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v1, v5
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v20, v19, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v11, v19, v10
+; GISEL-NEXT: v_mad_u64_u32 v[14:15], s[6:7], v21, v17, v[12:13]
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, -1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v16, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v15
-; GISEL-NEXT: v_cndmask_b32_e64 v5, v14, v17, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v14, vcc, 0, v18, vcc
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, 1, v11
-; GISEL-NEXT: v_addc_u32_e32 v17, vcc, 0, v14, vcc
+; GISEL-NEXT: v_mul_lo_u32 v12, v17, v14
+; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT: v_mul_hi_u32 v12, v17, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[6:7]
+; GISEL-NEXT: v_mul_hi_u32 v10, v19, v10
+; GISEL-NEXT: v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[8:9]
+; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[8:9], v16, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, 1, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[8:9]
+; GISEL-NEXT: v_addc_u32_e32 v12, vcc, 0, v18, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v1, v1, v15, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, 1, v11
+; GISEL-NEXT: v_addc_u32_e32 v16, vcc, 0, v12, vcc
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v11, v15, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[6:7]
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_mul_lo_u32 v13, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v13, v17, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v14, v19, v14
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v15, v13
+; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v13, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v14, v11
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v17, v10
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v19, v11, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v20, v13, 0
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[6:7]
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v14, v14, v17, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_mul_lo_u32 v12, v19, v13
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v12, v0
-; GISEL-NEXT: v_mul_hi_u32 v12, v8, v13
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_mul_hi_u32 v13, v19, v13
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v16, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v18, v12, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v14, v[11:12]
+; GISEL-NEXT: v_xor_b32_e32 v11, v5, v9
+; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v13, v[0:1]
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v15
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v3, v15, vcc
+; GISEL-NEXT: v_xor_b32_e32 v16, v0, v15
+; GISEL-NEXT: v_mul_lo_u32 v0, v14, v10
+; GISEL-NEXT: v_mul_lo_u32 v2, v13, v4
+; GISEL-NEXT: v_xor_b32_e32 v17, v1, v15
+; GISEL-NEXT: v_mul_hi_u32 v1, v13, v10
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v10
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v1, v14, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v2, v0
+; GISEL-NEXT: v_mul_hi_u32 v2, v13, v4
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v4
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v12, v1
-; GISEL-NEXT: v_add_i32_e32 v1, vcc, v13, v1
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v0
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v19, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v20, v8, 0
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
-; GISEL-NEXT: v_cndmask_b32_e32 v5, v15, v11, vcc
-; GISEL-NEXT: v_ashrrev_i32_e32 v15, 31, v3
-; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v20, v13, v[1:2]
-; GISEL-NEXT: v_xor_b32_e32 v1, v7, v4
-; GISEL-NEXT: v_xor_b32_e32 v7, v5, v1
-; GISEL-NEXT: v_cndmask_b32_e32 v14, v18, v14, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v21, v8, v[11:12]
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v15
-; GISEL-NEXT: v_addc_u32_e32 v3, vcc, v3, v15, vcc
-; GISEL-NEXT: v_xor_b32_e32 v11, v2, v15
-; GISEL-NEXT: v_mul_lo_u32 v2, v13, v0
-; GISEL-NEXT: v_mul_lo_u32 v5, v8, v4
-; GISEL-NEXT: v_xor_b32_e32 v12, v3, v15
-; GISEL-NEXT: v_mul_hi_u32 v3, v8, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v13, v0
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v2, v1
+; GISEL-NEXT: v_add_i32_e32 v1, vcc, v3, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v13, v0
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, v14, v1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v2, v17, v0
+; GISEL-NEXT: v_mul_lo_u32 v3, v16, v1
+; GISEL-NEXT: v_mul_hi_u32 v4, v16, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v17, v0
+; GISEL-NEXT: v_xor_b32_e32 v10, v12, v9
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v3, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v5, v2
-; GISEL-NEXT: v_mul_hi_u32 v5, v8, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GISEL-NEXT: v_add_i32_e32 v2, vcc, v2, v4
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v4, v17, v1
; GISEL-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, v4, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
-; GISEL-NEXT: v_addc_u32_e32 v2, vcc, v13, v2, vcc
-; GISEL-NEXT: v_mul_lo_u32 v3, v12, v0
-; GISEL-NEXT: v_mul_lo_u32 v4, v11, v2
-; GISEL-NEXT: v_mul_hi_u32 v5, v11, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v12, v0
-; GISEL-NEXT: v_xor_b32_e32 v8, v14, v1
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT: v_mul_hi_u32 v3, v16, v1
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v3
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, v12, v2
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; GISEL-NEXT: v_mul_hi_u32 v4, v11, v2
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v5, v0
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v5, v4
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, v0, v3
-; GISEL-NEXT: v_mul_hi_u32 v5, v12, v2
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v13, 0
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v3
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v0, v2
+; GISEL-NEXT: v_mul_hi_u32 v1, v17, v1
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v8, v12, 0
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
-; GISEL-NEXT: v_add_i32_e32 v14, vcc, v5, v0
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v14, v[3:4]
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v7, v1
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v8, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v9, v13, v[4:5]
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v11, v2
-; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v12, v7, vcc
-; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v12, v7
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v9
-; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v9, vcc
+; GISEL-NEXT: v_add_i32_e32 v13, vcc, v1, v0
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v8, v13, v[3:4]
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v11, v9
+; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v10, v9, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v7, v12, v[4:5]
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v16, v2
+; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], v17, v9, vcc
+; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v17, v9
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v3, v7
+; GISEL-NEXT: v_subb_u32_e32 v4, vcc, v4, v7, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v10
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[4:5]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v9
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v2, v8
+; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], v3, v7
; GISEL-NEXT: v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, v7, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v13
-; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v14, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v4, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v3, v5, v9, s[4:5]
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v12
+; GISEL-NEXT: v_addc_u32_e32 v9, vcc, 0, v13, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v4, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v8
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v4, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
; GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v5
-; GISEL-NEXT: v_addc_u32_e32 v8, vcc, 0, v7, vcc
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v9, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
; GISEL-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v4, v9, v7, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v13, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v14, v4, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v4, vcc
; GISEL-NEXT: v_xor_b32_e32 v4, v15, v6
; GISEL-NEXT: v_xor_b32_e32 v2, v2, v4
; GISEL-NEXT: v_xor_b32_e32 v3, v3, v4
@@ -2514,144 +2514,144 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v9, 0
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, 0, v1
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v9
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v5, 0
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v5
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_subb_u32_e64 v13, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v2, 0xffffff, v2
; GISEL-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
; GISEL-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
; GISEL-NEXT: v_trunc_f32_e32 v4, v4
; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v10, v3
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v10, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v13, v[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[7:8]
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v3
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v3
-; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
-; GISEL-NEXT: v_mul_lo_u32 v8, v10, v4
-; GISEL-NEXT: v_mul_lo_u32 v14, v13, v4
-; GISEL-NEXT: v_mul_hi_u32 v15, v10, v4
-; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT: v_cvt_u32_f32_e32 v11, v3
+; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v4
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v11, 0
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v14, v[4:5]
+; GISEL-NEXT: v_mul_lo_u32 v4, v14, v3
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v11, v[7:8]
+; GISEL-NEXT: v_mul_hi_u32 v7, v11, v3
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v3
+; GISEL-NEXT: v_mul_lo_u32 v8, v11, v9
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v9
+; GISEL-NEXT: v_mul_hi_u32 v15, v11, v9
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v8
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v14, v3
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v10, v3
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v15
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v3
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v10, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v13, v[4:5]
-; GISEL-NEXT: v_and_b32_e32 v11, 0xffffff, v0
-; GISEL-NEXT: v_mul_hi_u32 v0, v10, v3
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[7:8]
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v3
-; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
-; GISEL-NEXT: v_mul_lo_u32 v7, v10, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
+; GISEL-NEXT: v_mul_hi_u32 v8, v14, v9
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v7, v4
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v8, v4
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v3
+; GISEL-NEXT: v_addc_u32_e32 v14, vcc, v14, v4, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v11, 0
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v12, v14, v[4:5]
+; GISEL-NEXT: v_mul_lo_u32 v4, v14, v3
+; GISEL-NEXT: v_and_b32_e32 v12, 0xffffff, v0
+; GISEL-NEXT: v_mul_hi_u32 v0, v11, v3
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v11, v[7:8]
+; GISEL-NEXT: v_mul_hi_u32 v3, v14, v3
+; GISEL-NEXT: v_mul_lo_u32 v7, v11, v9
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v5, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v4
+; GISEL-NEXT: v_mul_lo_u32 v4, v14, v9
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v7, v0
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v5, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
+; GISEL-NEXT: v_mul_hi_u32 v7, v11, v9
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
+; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v14, v9
; GISEL-NEXT: v_add_i32_e32 v0, vcc, v3, v0
; GISEL-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v5, v3
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v4, v3
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v10, v0
-; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v13, v3, vcc
-; GISEL-NEXT: v_mul_lo_u32 v5, 0, v0
-; GISEL-NEXT: v_mul_lo_u32 v7, v11, v4
+; GISEL-NEXT: v_add_i32_e32 v3, vcc, v7, v3
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v11, v0
+; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v14, v3, vcc
+; GISEL-NEXT: v_mul_lo_u32 v7, 0, v0
+; GISEL-NEXT: v_mul_lo_u32 v8, v12, v4
; GISEL-NEXT: v_and_b32_e32 v3, 0xffffff, v6
-; GISEL-NEXT: v_mul_hi_u32 v6, v11, v0
+; GISEL-NEXT: v_mul_hi_u32 v6, v12, v0
; GISEL-NEXT: v_mul_hi_u32 v0, 0, v0
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
-; GISEL-NEXT: v_mul_hi_u32 v6, v11, v4
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v7, v0
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_mul_lo_u32 v8, 0, v4
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT: v_mul_hi_u32 v7, v12, v4
; GISEL-NEXT: v_cvt_f32_u32_e32 v10, v3
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v8, v0
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v7
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v6
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v1, v0, 0
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v1, v0, 0
+; GISEL-NEXT: v_mac_f32_e32 v10, 0x4f800000, v5
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v5, v10
; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 0, v3
-; GISEL-NEXT: v_mac_f32_e32 v10, 0x4f800000, v9
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v4, v[6:7]
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v6, v10
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v0, v[7:8]
-; GISEL-NEXT: v_mul_f32_e32 v6, 0x5f7ffffc, v6
-; GISEL-NEXT: v_mul_f32_e32 v7, 0x2f800000, v6
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v4, v[7:8]
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT: v_mul_f32_e32 v7, 0x2f800000, v5
; GISEL-NEXT: v_trunc_f32_e32 v7, v7
-; GISEL-NEXT: v_mac_f32_e32 v6, 0xcf800000, v7
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v6
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], 0, v0, v[8:9]
+; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v7
+; GISEL-NEXT: v_cvt_u32_f32_e32 v11, v5
; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v7
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v11, v5
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v14, v12, 0
-; GISEL-NEXT: v_subb_u32_e64 v5, s[4:5], 0, v9, vcc
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], 0, v9
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v13, v[7:8]
-; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v16, v1
+; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v11, 0
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v12, v6
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v13, v[8:9]
+; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], 0, v10
+; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v12, v1
; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v15, v12, v[10:11]
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v6
-; GISEL-NEXT: v_mul_hi_u32 v10, v12, v6
-; GISEL-NEXT: v_mul_lo_u32 v8, v12, v7
-; GISEL-NEXT: v_mul_hi_u32 v6, v13, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v11, -1, v17, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v11, v[5:6]
+; GISEL-NEXT: v_mul_lo_u32 v5, v13, v7
+; GISEL-NEXT: v_subbrev_u32_e32 v9, vcc, 0, v10, vcc
+; GISEL-NEXT: v_mul_lo_u32 v6, v11, v8
+; GISEL-NEXT: v_mul_hi_u32 v10, v11, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v13, v7
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v10
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v10, v13, v7
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_mul_hi_u32 v8, v12, v7
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v10, v6
+; GISEL-NEXT: v_mul_lo_u32 v10, v13, v8
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
+; GISEL-NEXT: v_mul_hi_u32 v6, v11, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
-; GISEL-NEXT: v_mul_hi_u32 v7, v13, v7
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v8
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v12, v5
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v11, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v16, -1, v17, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v6, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v12, 0
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, v16, v1
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v11, 0
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v12, v1
; GISEL-NEXT: v_subbrev_u32_e32 v17, vcc, 0, v9, vcc
; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v13, v[6:7]
; GISEL-NEXT: v_add_i32_e32 v14, vcc, 1, v0
; GISEL-NEXT: v_mul_lo_u32 v6, v13, v5
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v12, v[7:8]
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v15, v11, v[7:8]
; GISEL-NEXT: v_addc_u32_e32 v18, vcc, 0, v4, vcc
-; GISEL-NEXT: v_mul_hi_u32 v8, v12, v5
-; GISEL-NEXT: v_mul_lo_u32 v7, v12, v9
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v16, v1
+; GISEL-NEXT: v_mul_hi_u32 v8, v11, v5
+; GISEL-NEXT: v_mul_lo_u32 v7, v11, v9
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v12, v1
; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v17
; GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v1, vcc
@@ -2662,7 +2662,7 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_mul_lo_u32 v8, v13, v9
; GISEL-NEXT: v_mul_hi_u32 v5, v13, v5
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_mul_hi_u32 v7, v12, v9
+; GISEL-NEXT: v_mul_hi_u32 v7, v11, v9
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
@@ -2673,32 +2673,32 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v12, v5
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v11, v5
; GISEL-NEXT: v_addc_u32_e32 v6, vcc, v13, v6, vcc
; GISEL-NEXT: v_mul_lo_u32 v7, 0, v5
; GISEL-NEXT: v_mul_lo_u32 v8, v2, v6
-; GISEL-NEXT: v_mul_hi_u32 v12, v2, v5
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v5
; GISEL-NEXT: v_add_i32_e32 v9, vcc, 1, v14
; GISEL-NEXT: v_addc_u32_e32 v10, vcc, 0, v18, vcc
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
; GISEL-NEXT: v_mul_lo_u32 v8, 0, v6
; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v12
-; GISEL-NEXT: v_mul_hi_u32 v12, v2, v6
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v11
+; GISEL-NEXT: v_mul_hi_u32 v11, v2, v6
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v12
-; GISEL-NEXT: v_add_i32_e32 v12, vcc, v5, v7
-; GISEL-NEXT: v_mul_hi_u32 v13, 0, v6
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v3, v12, 0
+; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v11
+; GISEL-NEXT: v_add_i32_e32 v11, vcc, v5, v7
+; GISEL-NEXT: v_mul_hi_u32 v12, 0, v6
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v3, v11, 0
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GISEL-NEXT: v_cndmask_b32_e32 v1, v14, v9, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v9, v18, v10, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v13, v[6:7]
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v3, v12, v[6:7]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v12, v[7:8]
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v11, v[7:8]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], 0, v9
; GISEL-NEXT: v_subb_u32_e64 v4, s[4:5], 0, v9, vcc
@@ -2709,8 +2709,8 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v4
; GISEL-NEXT: v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v4, -1, v6, s[4:5]
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, 1, v12
-; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v13, vcc
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, 1, v11
+; GISEL-NEXT: v_addc_u32_e32 v7, vcc, 0, v12, vcc
; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3
; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v5
@@ -2721,8 +2721,8 @@ define <2 x i64> @v_sdiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v12, v3, vcc
; GISEL-NEXT: s_setpc_b64 s[30:31]
;
; CGP-LABEL: v_sdiv_v2i64_24bit:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index 3ab7d91543084..a2367cc9e3eea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -1343,7 +1343,6 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
; GFX8-NEXT: v_addc_u32_e32 v7, vcc, v7, v1, vcc
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s16, v6, 0
-; GFX8-NEXT: v_mov_b32_e32 v8, s11
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s16, v7, v[1:2]
; GFX8-NEXT: v_mul_lo_u32 v1, v7, v0
; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s17, v6, v[2:3]
@@ -1373,6 +1372,7 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_lo_u32 v3, s10, v1
; GFX8-NEXT: v_mul_hi_u32 v4, s10, v0
; GFX8-NEXT: v_mul_hi_u32 v0, s11, v0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
@@ -1385,135 +1385,136 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v4, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v0, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v0, v2
; GFX8-NEXT: v_mul_hi_u32 v4, s11, v1
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v6, 0
+; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v8, 0
; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v4, v2
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s6, v7, v[1:2]
-; GFX8-NEXT: v_sub_u32_e32 v9, vcc, s10, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s7, v6, v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v4, v2
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s6, v9, v[1:2]
+; GFX8-NEXT: v_mov_b32_e32 v1, s11
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s10, v0
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s7, v8, v[2:3]
; GFX8-NEXT: s_ashr_i32 s10, s3, 31
; GFX8-NEXT: s_add_u32 s18, s18, s8
; GFX8-NEXT: s_addc_u32 s19, s19, s8
-; GFX8-NEXT: v_subb_u32_e64 v8, s[0:1], v8, v4, vcc
-; GFX8-NEXT: v_sub_u32_e64 v0, s[0:1], s11, v4
-; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v8
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v9
-; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v0, v1, vcc
+; GFX8-NEXT: v_subb_u32_e64 v1, s[0:1], v1, v4, vcc
+; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s11, v4
+; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v1
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v8
-; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s6, v9
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v2, v3, s[0:1]
-; GFX8-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v0, vcc
-; GFX8-NEXT: v_add_u32_e64 v4, s[0:1], 1, v6
-; GFX8-NEXT: v_addc_u32_e64 v5, s[0:1], 0, v7, s[0:1]
+; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v0
+; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v2, v6, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v1
+; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s6, v0
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v3, v4, s[0:1]
+; GFX8-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v2, vcc
+; GFX8-NEXT: v_add_u32_e64 v7, s[0:1], 1, v8
+; GFX8-NEXT: v_addc_u32_e64 v13, s[0:1], 0, v9, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v12
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v11
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
+; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v11
+; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v12
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[0:1]
; GFX8-NEXT: s_add_u32 s0, s2, s10
; GFX8-NEXT: s_mov_b32 s11, s10
; GFX8-NEXT: s_addc_u32 s1, s3, s10
; GFX8-NEXT: s_xor_b64 s[16:17], s[0:1], s[10:11]
-; GFX8-NEXT: v_cvt_f32_u32_e32 v13, s17
+; GFX8-NEXT: v_cvt_f32_u32_e32 v5, s17
; GFX8-NEXT: v_cvt_f32_u32_e32 v3, s16
; GFX8-NEXT: s_xor_b64 s[18:19], s[18:19], s[8:9]
-; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v4
-; GFX8-NEXT: v_mul_f32_e32 v13, 0x4f800000, v13
-; GFX8-NEXT: v_add_f32_e32 v3, v13, v3
+; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v7
+; GFX8-NEXT: v_mul_f32_e32 v5, 0x4f800000, v5
+; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
; GFX8-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX8-NEXT: v_subb_u32_e32 v13, vcc, v0, v1, vcc
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v2, v6, vcc
; GFX8-NEXT: s_sub_u32 s5, 0, s16
-; GFX8-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v3
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX8-NEXT: v_trunc_f32_e32 v1, v1
-; GFX8-NEXT: v_mul_f32_e32 v3, 0xcf800000, v1
-; GFX8-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX8-NEXT: v_cvt_u32_f32_e32 v16, v0
-; GFX8-NEXT: v_addc_u32_e64 v15, s[0:1], 0, v5, s[0:1]
-; GFX8-NEXT: v_cvt_u32_f32_e32 v17, v1
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v16, 0
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v3
+; GFX8-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX8-NEXT: v_trunc_f32_e32 v3, v3
+; GFX8-NEXT: v_mul_f32_e32 v6, 0xcf800000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v6, v2
+; GFX8-NEXT: v_cvt_u32_f32_e32 v16, v2
+; GFX8-NEXT: v_addc_u32_e64 v15, s[0:1], 0, v13, s[0:1]
+; GFX8-NEXT: v_cvt_u32_f32_e32 v17, v3
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s5, v16, 0
; GFX8-NEXT: v_subrev_u32_e32 v18, vcc, s6, v11
-; GFX8-NEXT: v_subbrev_u32_e32 v13, vcc, 0, v13, vcc
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s5, v17, v[1:2]
+; GFX8-NEXT: v_subbrev_u32_e32 v19, vcc, 0, v5, vcc
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s5, v17, v[3:4]
; GFX8-NEXT: s_subb_u32 s6, 0, s17
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v14, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v14, v5, v15, vcc
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s6, v16, v[2:3]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v14, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; GFX8-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s6, v16, v[4:5]
; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v10
-; GFX8-NEXT: v_cndmask_b32_e64 v5, v6, v1, s[0:1]
-; GFX8-NEXT: v_mul_lo_u32 v1, v17, v0
-; GFX8-NEXT: v_mul_lo_u32 v2, v16, v4
-; GFX8-NEXT: v_mul_hi_u32 v3, v16, v0
-; GFX8-NEXT: v_mul_hi_u32 v0, v17, v0
-; GFX8-NEXT: v_cndmask_b32_e64 v6, v7, v14, s[0:1]
-; GFX8-NEXT: v_add_u32_e64 v1, s[2:3], v1, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; GFX8-NEXT: v_add_u32_e64 v1, s[2:3], v1, v3
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX8-NEXT: v_mul_lo_u32 v3, v17, v4
-; GFX8-NEXT: v_add_u32_e64 v1, s[2:3], v2, v1
-; GFX8-NEXT: v_mul_hi_u32 v2, v16, v4
-; GFX8-NEXT: v_add_u32_e64 v0, s[2:3], v3, v0
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v3, s[0:1]
+; GFX8-NEXT: v_mul_lo_u32 v3, v17, v2
+; GFX8-NEXT: v_mul_lo_u32 v7, v16, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v13, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v9, v16, v2
+; GFX8-NEXT: v_mul_hi_u32 v2, v17, v2
+; GFX8-NEXT: v_add_u32_e64 v3, s[2:3], v3, v7
+; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
+; GFX8-NEXT: v_add_u32_e64 v3, s[2:3], v3, v9
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3]
-; GFX8-NEXT: v_add_u32_e64 v0, s[2:3], v0, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; GFX8-NEXT: v_add_u32_e64 v2, s[2:3], v3, v2
-; GFX8-NEXT: v_mul_hi_u32 v3, v17, v4
-; GFX8-NEXT: v_add_u32_e64 v0, s[2:3], v0, v1
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX8-NEXT: v_add_u32_e64 v1, s[2:3], v2, v1
-; GFX8-NEXT: v_add_u32_e64 v1, s[2:3], v3, v1
-; GFX8-NEXT: v_add_u32_e64 v10, s[2:3], v16, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v11, v18, vcc
-; GFX8-NEXT: v_addc_u32_e64 v11, s[2:3], v17, v1, s[2:3]
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s5, v10, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v0, s[0:1]
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v11, v[3:4]
-; GFX8-NEXT: v_xor_b32_e32 v5, s20, v5
-; GFX8-NEXT: v_xor_b32_e32 v6, s21, v6
-; GFX8-NEXT: v_mov_b32_e32 v9, s21
-; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s6, v10, v[0:1]
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s20, v5
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v6, v9, vcc
-; GFX8-NEXT: v_mul_lo_u32 v5, v11, v2
-; GFX8-NEXT: v_mul_lo_u32 v6, v10, v3
-; GFX8-NEXT: v_xor_b32_e32 v4, s4, v7
-; GFX8-NEXT: v_mul_hi_u32 v7, v10, v2
-; GFX8-NEXT: v_mul_hi_u32 v2, v11, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v7
+; GFX8-NEXT: v_mul_lo_u32 v9, v17, v6
+; GFX8-NEXT: v_add_u32_e64 v3, s[2:3], v7, v3
+; GFX8-NEXT: v_mul_hi_u32 v7, v16, v6
+; GFX8-NEXT: v_add_u32_e64 v2, s[2:3], v9, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[2:3]
+; GFX8-NEXT: v_add_u32_e64 v2, s[2:3], v2, v7
+; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
+; GFX8-NEXT: v_add_u32_e64 v7, s[2:3], v9, v7
+; GFX8-NEXT: v_mul_hi_u32 v6, v17, v6
+; GFX8-NEXT: v_add_u32_e64 v2, s[2:3], v2, v3
+; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3]
+; GFX8-NEXT: v_add_u32_e64 v3, s[2:3], v7, v3
+; GFX8-NEXT: v_add_u32_e64 v3, s[2:3], v6, v3
+; GFX8-NEXT: v_add_u32_e64 v6, s[2:3], v16, v2
+; GFX8-NEXT: v_addc_u32_e64 v7, s[2:3], v17, v3, s[2:3]
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s5, v6, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v8, v11, v18, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v9, v12, v19, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v9, v1, v9, s[0:1]
+; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v7, v[3:4]
+; GFX8-NEXT: v_xor_b32_e32 v10, s20, v4
+; GFX8-NEXT: v_xor_b32_e32 v5, s21, v5
+; GFX8-NEXT: v_mov_b32_e32 v11, s21
+; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s6, v6, v[0:1]
+; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s20, v10
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v5, v11, vcc
+; GFX8-NEXT: v_xor_b32_e32 v4, s4, v8
+; GFX8-NEXT: v_mul_lo_u32 v5, v7, v2
+; GFX8-NEXT: v_mul_lo_u32 v8, v6, v3
+; GFX8-NEXT: v_mul_hi_u32 v10, v6, v2
+; GFX8-NEXT: v_mul_hi_u32 v2, v7, v2
+; GFX8-NEXT: v_xor_b32_e32 v9, s4, v9
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v8
+; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v10
; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX8-NEXT: v_mul_lo_u32 v7, v11, v3
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v6, v5
-; GFX8-NEXT: v_mul_hi_u32 v6, v10, v3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v7, v6
-; GFX8-NEXT: v_mul_hi_u32 v3, v11, v3
+; GFX8-NEXT: v_mul_lo_u32 v10, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v8, v5
+; GFX8-NEXT: v_mul_hi_u32 v8, v6, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v10, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v8
+; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v10, v8
+; GFX8-NEXT: v_mul_hi_u32 v3, v7, v3
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5
; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v6, v5
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v8, v5
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v5
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v10, v2
-; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v11, v3, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, v6, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v7, v3, vcc
; GFX8-NEXT: v_mul_lo_u32 v5, s19, v2
; GFX8-NEXT: v_mul_lo_u32 v6, s18, v3
; GFX8-NEXT: v_mul_hi_u32 v7, s18, v2
; GFX8-NEXT: v_mul_hi_u32 v2, s19, v2
-; GFX8-NEXT: v_xor_b32_e32 v8, s4, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, s4
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v6
; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v7
@@ -1533,9 +1534,8 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v6, v5
; GFX8-NEXT: v_add_u32_e32 v11, vcc, v7, v5
; GFX8-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s16, v11, v[3:4]
-; GFX8-NEXT: v_mov_b32_e32 v9, s4
; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v8, v9, vcc
+; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v9, v8, vcc
; GFX8-NEXT: v_mad_u64_u32 v[8:9], s[0:1], s17, v10, v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v12, s19
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s18, v2
@@ -1652,7 +1652,6 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v0
; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v7, v1, vcc
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s16, v6, 0
-; GFX9-NEXT: v_mov_b32_e32 v8, s11
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s16, v7, v[1:2]
; GFX9-NEXT: v_mul_lo_u32 v1, v7, v0
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s17, v6, v[2:3]
@@ -1689,135 +1688,137 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mul_lo_u32 v4, s11, v1
; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
; GFX9-NEXT: v_mul_hi_u32 v3, s10, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3
; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v0, v2
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v6, 0
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v0, v2
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v8, 0
; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX9-NEXT: v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT: v_add3_u32 v7, v3, v2, v5
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s6, v7, v[1:2]
-; GFX9-NEXT: v_sub_co_u32_e32 v9, vcc, s10, v0
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s7, v6, v[2:3]
+; GFX9-NEXT: v_add3_u32 v9, v3, v2, v5
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s6, v9, v[1:2]
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s10, v0
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s7, v8, v[2:3]
; GFX9-NEXT: s_ashr_i32 s10, s3, 31
; GFX9-NEXT: s_add_u32 s18, s18, s8
; GFX9-NEXT: s_addc_u32 s19, s19, s8
-; GFX9-NEXT: v_subb_co_u32_e64 v8, s[0:1], v8, v4, vcc
-; GFX9-NEXT: v_sub_u32_e32 v0, s11, v4
-; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v8
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v9
-; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v1, vcc
+; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v4, vcc
+; GFX9-NEXT: v_sub_u32_e32 v2, s11, v4
+; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v1
; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v8
-; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s6, v9
-; GFX9-NEXT: v_cndmask_b32_e64 v10, v2, v3, s[0:1]
-; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[0:1], 0, v0, vcc
-; GFX9-NEXT: v_add_co_u32_e64 v4, s[0:1], 1, v6
-; GFX9-NEXT: v_addc_co_u32_e64 v5, s[0:1], 0, v7, s[0:1]
+; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v0
+; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v6, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s6, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v3, v4, s[0:1]
+; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[0:1], 0, v2, vcc
+; GFX9-NEXT: v_add_co_u32_e64 v7, s[0:1], 1, v8
+; GFX9-NEXT: v_addc_co_u32_e64 v13, s[0:1], 0, v9, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v12
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v11
; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
+; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v11
+; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v12
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[0:1]
; GFX9-NEXT: s_add_u32 s0, s2, s10
; GFX9-NEXT: s_mov_b32 s11, s10
; GFX9-NEXT: s_addc_u32 s1, s3, s10
; GFX9-NEXT: s_xor_b64 s[16:17], s[0:1], s[10:11]
-; GFX9-NEXT: v_cvt_f32_u32_e32 v13, s17
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s17
; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s16
; GFX9-NEXT: s_xor_b64 s[18:19], s[18:19], s[8:9]
-; GFX9-NEXT: v_add_co_u32_e64 v14, s[0:1], 1, v4
-; GFX9-NEXT: v_mul_f32_e32 v13, 0x4f800000, v13
-; GFX9-NEXT: v_add_f32_e32 v3, v13, v3
+; GFX9-NEXT: v_add_co_u32_e64 v14, s[0:1], 1, v7
+; GFX9-NEXT: v_mul_f32_e32 v5, 0x4f800000, v5
+; GFX9-NEXT: v_add_f32_e32 v3, v5, v3
; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v0, v1, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v2, v6, vcc
; GFX9-NEXT: s_sub_u32 s5, 0, s16
-; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v3
-; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT: v_trunc_f32_e32 v1, v1
-; GFX9-NEXT: v_mul_f32_e32 v3, 0xcf800000, v1
-; GFX9-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v16, v0
-; GFX9-NEXT: v_addc_co_u32_e64 v15, s[0:1], 0, v5, s[0:1]
-; GFX9-NEXT: v_cvt_u32_f32_e32 v17, v1
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v16, 0
+; GFX9-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v3
+; GFX9-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
+; GFX9-NEXT: v_trunc_f32_e32 v3, v3
+; GFX9-NEXT: v_mul_f32_e32 v6, 0xcf800000, v3
+; GFX9-NEXT: v_add_f32_e32 v2, v6, v2
+; GFX9-NEXT: v_cvt_u32_f32_e32 v16, v2
+; GFX9-NEXT: v_addc_co_u32_e64 v15, s[0:1], 0, v13, s[0:1]
+; GFX9-NEXT: v_cvt_u32_f32_e32 v17, v3
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s5, v16, 0
; GFX9-NEXT: v_subrev_co_u32_e32 v18, vcc, s6, v11
-; GFX9-NEXT: v_subbrev_co_u32_e32 v13, vcc, 0, v13, vcc
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s5, v17, v[1:2]
+; GFX9-NEXT: v_subbrev_co_u32_e32 v19, vcc, 0, v5, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s5, v17, v[3:4]
; GFX9-NEXT: s_subb_u32 s6, 0, s17
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v14, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v14, v5, v15, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s6, v16, v[2:3]
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v14, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s6, v16, v[4:5]
; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v10
-; GFX9-NEXT: v_cndmask_b32_e64 v5, v6, v1, s[0:1]
-; GFX9-NEXT: v_mul_lo_u32 v1, v17, v0
-; GFX9-NEXT: v_mul_lo_u32 v2, v16, v4
-; GFX9-NEXT: v_mul_hi_u32 v3, v16, v0
-; GFX9-NEXT: v_mul_hi_u32 v0, v17, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, v14, s[0:1]
-; GFX9-NEXT: v_add_co_u32_e64 v1, s[2:3], v1, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; GFX9-NEXT: v_add_co_u32_e64 v1, s[2:3], v1, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX9-NEXT: v_mul_lo_u32 v3, v17, v4
-; GFX9-NEXT: v_add_u32_e32 v1, v2, v1
-; GFX9-NEXT: v_mul_hi_u32 v2, v16, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v18, vcc
-; GFX9-NEXT: v_add_co_u32_e64 v0, s[2:3], v3, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v3, s[0:1]
+; GFX9-NEXT: v_mul_lo_u32 v3, v17, v2
+; GFX9-NEXT: v_mul_lo_u32 v7, v16, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v13, s[0:1]
+; GFX9-NEXT: v_mul_hi_u32 v9, v16, v2
+; GFX9-NEXT: v_mul_hi_u32 v2, v17, v2
+; GFX9-NEXT: v_add_co_u32_e64 v3, s[2:3], v3, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
+; GFX9-NEXT: v_add_co_u32_e64 v3, s[2:3], v3, v9
; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3]
-; GFX9-NEXT: v_add_co_u32_e64 v0, s[2:3], v0, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; GFX9-NEXT: v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT: v_mul_hi_u32 v3, v17, v4
-; GFX9-NEXT: v_add_co_u32_e64 v0, s[2:3], v0, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX9-NEXT: v_add3_u32 v1, v2, v1, v3
-; GFX9-NEXT: v_add_co_u32_e64 v10, s[2:3], v16, v0
-; GFX9-NEXT: v_addc_co_u32_e64 v11, s[2:3], v17, v1, s[2:3]
-; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s5, v10, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v7, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v0, s[0:1]
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v11, v[3:4]
-; GFX9-NEXT: v_xor_b32_e32 v5, s20, v5
-; GFX9-NEXT: v_xor_b32_e32 v6, s21, v6
-; GFX9-NEXT: v_mov_b32_e32 v9, s21
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s6, v10, v[0:1]
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s20, v5
-; GFX9-NEXT: v_mul_lo_u32 v4, v11, v2
-; GFX9-NEXT: v_mul_lo_u32 v5, v10, v3
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v6, v9, vcc
-; GFX9-NEXT: v_mul_hi_u32 v6, v10, v2
+; GFX9-NEXT: v_mul_lo_u32 v9, v17, v6
+; GFX9-NEXT: v_add_u32_e32 v3, v7, v3
+; GFX9-NEXT: v_mul_hi_u32 v7, v16, v6
+; GFX9-NEXT: v_mul_hi_u32 v6, v17, v6
+; GFX9-NEXT: v_add_co_u32_e64 v2, s[2:3], v9, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[2:3]
+; GFX9-NEXT: v_add_co_u32_e64 v2, s[2:3], v2, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
+; GFX9-NEXT: v_add_co_u32_e64 v2, s[2:3], v2, v3
+; GFX9-NEXT: v_add_u32_e32 v7, v9, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3]
+; GFX9-NEXT: v_add3_u32 v3, v7, v3, v6
+; GFX9-NEXT: v_add_co_u32_e64 v6, s[2:3], v16, v2
+; GFX9-NEXT: v_addc_co_u32_e64 v7, s[2:3], v17, v3, s[2:3]
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s5, v6, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v11, v18, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v12, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v8, v0, v8, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v9, v1, v9, s[0:1]
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s5, v7, v[3:4]
+; GFX9-NEXT: v_xor_b32_e32 v10, s20, v4
+; GFX9-NEXT: v_xor_b32_e32 v5, s21, v5
+; GFX9-NEXT: v_mov_b32_e32 v11, s21
+; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s6, v6, v[0:1]
+; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s20, v10
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v5, v11, vcc
+; GFX9-NEXT: v_mul_lo_u32 v4, v7, v2
+; GFX9-NEXT: v_mul_lo_u32 v5, v6, v3
+; GFX9-NEXT: v_mul_hi_u32 v10, v6, v2
+; GFX9-NEXT: v_mul_hi_u32 v2, v7, v2
+; GFX9-NEXT: v_xor_b32_e32 v8, s4, v8
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6
+; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v10
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX9-NEXT: v_mul_lo_u32 v6, v11, v3
-; GFX9-NEXT: v_mul_hi_u32 v2, v11, v2
+; GFX9-NEXT: v_mul_lo_u32 v10, v7, v3
; GFX9-NEXT: v_add_u32_e32 v4, v5, v4
-; GFX9-NEXT: v_mul_hi_u32 v5, v10, v3
-; GFX9-NEXT: v_mul_hi_u32 v3, v11, v3
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2
-; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT: v_mul_hi_u32 v5, v6, v3
+; GFX9-NEXT: v_mul_hi_u32 v3, v7, v3
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9-NEXT: v_add_u32_e32 v5, v6, v5
+; GFX9-NEXT: v_add_u32_e32 v5, v10, v5
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX9-NEXT: v_add3_u32 v3, v5, v4, v3
-; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v10, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v11, v3, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2
+; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v7, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v4, s19, v2
; GFX9-NEXT: v_mul_lo_u32 v5, s18, v3
; GFX9-NEXT: v_mul_hi_u32 v6, s18, v2
; GFX9-NEXT: v_mul_hi_u32 v2, s19, v2
-; GFX9-NEXT: v_xor_b32_e32 v9, s4, v7
+; GFX9-NEXT: v_mul_hi_u32 v7, s19, v3
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6
@@ -1825,7 +1826,7 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mul_lo_u32 v6, s19, v3
; GFX9-NEXT: v_add_u32_e32 v4, v5, v4
; GFX9-NEXT: v_mul_hi_u32 v5, s18, v3
-; GFX9-NEXT: v_mul_hi_u32 v7, s19, v3
+; GFX9-NEXT: v_xor_b32_e32 v9, s4, v9
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
@@ -1836,10 +1837,9 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_add_u32_e32 v5, v6, v5
; GFX9-NEXT: v_add3_u32 v12, v5, v4, v7
; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s16, v12, v[3:4]
-; GFX9-NEXT: v_xor_b32_e32 v8, s4, v8
; GFX9-NEXT: v_mov_b32_e32 v11, s4
-; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s4, v9
-; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v8, v11, vcc
+; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s4, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v9, v11, vcc
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[0:1], s17, v10, v[6:7]
; GFX9-NEXT: v_mov_b32_e32 v11, s19
; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, s18, v2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 970d241b44458..b001783de832f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -3005,151 +3005,151 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v4
; GISEL-NEXT: v_cvt_f32_u32_e32 v3, v1
-; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v9, 0
-; GISEL-NEXT: v_sub_i32_e32 v11, vcc, 0, v1
-; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v9
+; GISEL-NEXT: v_cvt_f32_ubyte0_e32 v4, 0
+; GISEL-NEXT: v_sub_i32_e32 v13, vcc, 0, v1
+; GISEL-NEXT: v_mac_f32_e32 v3, 0x4f800000, v4
; GISEL-NEXT: v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_subb_u32_e64 v14, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0
; GISEL-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v3
-; GISEL-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3
-; GISEL-NEXT: v_trunc_f32_e32 v4, v4
-; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v4
-; GISEL-NEXT: v_cvt_u32_f32_e32 v10, v3
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
+; GISEL-NEXT: v_mul_f32_e32 v5, 0x2f800000, v3
+; GISEL-NEXT: v_trunc_f32_e32 v5, v5
+; GISEL-NEXT: v_mac_f32_e32 v3, 0xcf800000, v5
+; GISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GISEL-NEXT: v_cvt_u32_f32_e32 v5, v5
; GISEL-NEXT: v_and_b32_e32 v2, 0xffffff, v2
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v10, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v13, v[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[7:8]
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v3
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v3
-; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
-; GISEL-NEXT: v_mul_lo_u32 v8, v10, v4
-; GISEL-NEXT: v_mul_lo_u32 v14, v13, v4
-; GISEL-NEXT: v_mul_hi_u32 v15, v10, v4
-; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v14, v3
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v15
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v3, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v5, v[8:9]
+; GISEL-NEXT: v_mul_lo_u32 v8, v5, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v3, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v9, v3, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT: v_mul_lo_u32 v10, v3, v11
+; GISEL-NEXT: v_mul_lo_u32 v12, v5, v11
+; GISEL-NEXT: v_mul_hi_u32 v15, v3, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v12, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v15
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT: v_mul_hi_u32 v10, v5, v11
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT: v_add_i32_e32 v10, vcc, v10, v3
-; GISEL-NEXT: v_addc_u32_e32 v13, vcc, v13, v4, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v11, v10, 0
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v11, v13, v[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[7:8]
-; GISEL-NEXT: v_mul_lo_u32 v5, v13, v3
-; GISEL-NEXT: v_mul_hi_u32 v8, v10, v3
-; GISEL-NEXT: v_mul_hi_u32 v3, v13, v3
-; GISEL-NEXT: v_mul_lo_u32 v7, v10, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v8, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; GISEL-NEXT: v_mul_hi_u32 v7, v10, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v8, v3
; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v7, vcc, v8, v7
-; GISEL-NEXT: v_mul_hi_u32 v4, v13, v4
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
-; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v10, v3
-; GISEL-NEXT: v_addc_u32_e32 v4, vcc, v13, v4, vcc
-; GISEL-NEXT: v_mul_lo_u32 v7, 0, v5
-; GISEL-NEXT: v_mul_lo_u32 v8, v0, v4
-; GISEL-NEXT: v_and_b32_e32 v3, 0xffffff, v6
-; GISEL-NEXT: v_mul_hi_u32 v6, v0, v5
-; GISEL-NEXT: v_mul_hi_u32 v5, 0, v5
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v13, v3, 0
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v13, v5, v[8:9]
+; GISEL-NEXT: v_mul_lo_u32 v8, v5, v7
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v3, v[9:10]
+; GISEL-NEXT: v_mul_hi_u32 v10, v3, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, v5, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v3, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_mul_lo_u32 v10, v5, v11
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_mul_hi_u32 v9, v3, v11
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v9, vcc, v10, v9
+; GISEL-NEXT: v_mul_hi_u32 v10, v5, v11
; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT: v_mul_lo_u32 v8, 0, v4
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GISEL-NEXT: v_mul_hi_u32 v7, v0, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v3, v7
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT: v_mul_lo_u32 v8, 0, v7
+; GISEL-NEXT: v_mul_lo_u32 v9, v0, v5
+; GISEL-NEXT: v_and_b32_e32 v3, 0xffffff, v6
+; GISEL-NEXT: v_mul_hi_u32 v6, v0, v7
+; GISEL-NEXT: v_mul_hi_u32 v7, 0, v7
+; GISEL-NEXT: v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT: v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT: v_add_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT: v_mul_hi_u32 v8, v0, v5
; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v8, v5
-; GISEL-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT: v_cvt_f32_u32_e32 v7, v3
-; GISEL-NEXT: v_add_i32_e32 v11, vcc, v5, v6
-; GISEL-NEXT: v_mul_hi_u32 v6, 0, v4
-; GISEL-NEXT: v_mac_f32_e32 v7, 0x4f800000, v9
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, v11, 0
-; GISEL-NEXT: v_rcp_iflag_f32_e32 v9, v7
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 0, v3
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v1, v6, v[5:6]
-; GISEL-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v9
-; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v9, v7
+; GISEL-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GISEL-NEXT: v_cvt_f32_u32_e32 v8, v3
+; GISEL-NEXT: v_add_i32_e32 v12, vcc, v7, v6
+; GISEL-NEXT: v_mul_hi_u32 v7, 0, v5
+; GISEL-NEXT: v_mac_f32_e32 v8, 0x4f800000, v4
+; GISEL-NEXT: v_rcp_iflag_f32_e32 v4, v8
+; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v1, v12, 0
+; GISEL-NEXT: v_sub_i32_e32 v15, vcc, 0, v3
+; GISEL-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v7, v[6:7]
+; GISEL-NEXT: v_mul_f32_e32 v6, 0x2f800000, v4
; GISEL-NEXT: v_trunc_f32_e32 v6, v6
-; GISEL-NEXT: v_mac_f32_e32 v5, 0xcf800000, v6
-; GISEL-NEXT: v_cvt_u32_f32_e32 v12, v5
-; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v6
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], 0, v11, v[7:8]
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v14, v12, 0
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, v9, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v14, v13, v[6:7]
-; GISEL-NEXT: v_mul_lo_u32 v4, v13, v5
-; GISEL-NEXT: v_sub_i32_e64 v6, s[4:5], 0, v9
-; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v15, v12, v[7:8]
-; GISEL-NEXT: v_mul_hi_u32 v9, v12, v5
+; GISEL-NEXT: v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT: v_cvt_u32_f32_e32 v13, v4
+; GISEL-NEXT: v_cvt_u32_f32_e32 v14, v6
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], 0, v12, v[8:9]
+; GISEL-NEXT: v_subb_u32_e64 v16, s[4:5], 0, 0, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v13, 0
+; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v5
+; GISEL-NEXT: v_subb_u32_e64 v11, s[4:5], 0, v10, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v14, v[7:8]
+; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], 0, v10
+; GISEL-NEXT: v_mul_hi_u32 v10, v13, v6
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v13, v[8:9]
+; GISEL-NEXT: v_mul_lo_u32 v5, v14, v6
; GISEL-NEXT: v_cmp_ge_u32_e64 s[4:5], v0, v1
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[4:5]
-; GISEL-NEXT: v_mul_lo_u32 v7, v12, v10
-; GISEL-NEXT: v_mul_hi_u32 v5, v13, v5
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v9
-; GISEL-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT: v_mul_lo_u32 v9, v13, v10
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v7, v4
-; GISEL-NEXT: v_mul_hi_u32 v7, v12, v10
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v9, v5
-; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v7
-; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v7, s[4:5], v9, v7
-; GISEL-NEXT: v_mul_hi_u32 v9, v13, v10
-; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v5, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT: v_mul_lo_u32 v8, v13, v4
+; GISEL-NEXT: v_mul_hi_u32 v6, v14, v6
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v5, v10
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[4:5]
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v7, v5
-; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v9, v5
-; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v12, v4
-; GISEL-NEXT: v_addc_u32_e64 v11, s[4:5], v13, v5, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v10, 0
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; GISEL-NEXT: v_cndmask_b32_e64 v12, -1, v8, s[4:5]
-; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v6, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v14, v11, v[5:6]
-; GISEL-NEXT: v_sub_i32_e32 v13, vcc, v0, v1
-; GISEL-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v15, v10, v[6:7]
-; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v13, v1
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v4
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v8, v5
+; GISEL-NEXT: v_mul_hi_u32 v8, v13, v4
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v10, v6
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v6, v8
+; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v8, s[4:5], v10, v8
+; GISEL-NEXT: v_mul_hi_u32 v4, v14, v4
+; GISEL-NEXT: v_add_i32_e64 v5, s[4:5], v6, v5
+; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; GISEL-NEXT: v_add_i32_e64 v6, s[4:5], v8, v6
+; GISEL-NEXT: v_add_i32_e64 v4, s[4:5], v4, v6
+; GISEL-NEXT: v_add_i32_e64 v10, s[4:5], v13, v5
+; GISEL-NEXT: v_addc_u32_e64 v12, s[4:5], v14, v4, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v15, v10, 0
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v11
+; GISEL-NEXT: v_cndmask_b32_e64 v13, -1, v9, s[4:5]
+; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v7, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v12, v[5:6]
+; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v0, v1
+; GISEL-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v8, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v16, v10, v[6:7]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v14, v1
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v6, v11, v4
+; GISEL-NEXT: v_mul_lo_u32 v6, v12, v4
; GISEL-NEXT: v_mul_lo_u32 v7, v10, v8
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v14
+; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v15
; GISEL-NEXT: v_cndmask_b32_e32 v9, -1, v5, vcc
; GISEL-NEXT: v_mul_hi_u32 v5, v10, v4
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GISEL-NEXT: v_mul_lo_u32 v6, v11, v8
-; GISEL-NEXT: v_mul_hi_u32 v4, v11, v4
+; GISEL-NEXT: v_mul_lo_u32 v6, v12, v8
+; GISEL-NEXT: v_mul_hi_u32 v4, v12, v4
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; GISEL-NEXT: v_mul_hi_u32 v7, v10, v8
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v6, v4
@@ -3157,18 +3157,18 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v7
; GISEL-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT: v_mul_hi_u32 v7, v11, v8
+; GISEL-NEXT: v_mul_hi_u32 v7, v12, v8
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v5
; GISEL-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; GISEL-NEXT: v_add_i32_e32 v5, vcc, v7, v5
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v10, v4
-; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v11, v5, vcc
+; GISEL-NEXT: v_addc_u32_e32 v5, vcc, v12, v5, vcc
; GISEL-NEXT: v_mul_lo_u32 v6, 0, v4
; GISEL-NEXT: v_mul_lo_u32 v7, v2, v5
; GISEL-NEXT: v_mul_hi_u32 v10, v2, v4
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v13, v1
-; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v14, vcc
+; GISEL-NEXT: v_sub_i32_e32 v1, vcc, v14, v1
+; GISEL-NEXT: v_subbrev_u32_e32 v8, vcc, 0, v15, vcc
; GISEL-NEXT: v_add_i32_e32 v6, vcc, v6, v7
; GISEL-NEXT: v_mul_lo_u32 v7, 0, v5
; GISEL-NEXT: v_mul_hi_u32 v4, 0, v4
@@ -3178,15 +3178,15 @@ define <2 x i64> @v_srem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v7, v4
; GISEL-NEXT: v_add_i32_e32 v4, vcc, v4, v10
; GISEL-NEXT: v_add_i32_e32 v10, vcc, v4, v6
-; GISEL-NEXT: v_mul_hi_u32 v11, 0, v5
+; GISEL-NEXT: v_mul_hi_u32 v12, 0, v5
; GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, v10, 0
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v14, v8, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v11, v[5:6]
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v14, v1, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v8, v15, v8, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, v12, v[5:6]
+; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
; GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v16, v8, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v1, v11, v8, vcc
; GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], 0, v10, v[6:7]
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
; GISEL-NEXT: v_sub_i32_e64 v5, s[4:5], 0, v8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index ddc3c2347f0ed..6a3074e598208 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -1272,7 +1272,7 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: v_mov_b32_e32 v9, s5
+; GFX9-NEXT: v_mov_b32_e32 v10, s5
; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0
; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
@@ -1352,84 +1352,83 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s4, v2, 0
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX9-NEXT: v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT: v_add3_u32 v10, v3, v5, v6
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s4, v10, v[1:2]
+; GFX9-NEXT: v_add3_u32 v3, v3, v5, v6
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s4, v3, v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s16, v0
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[0:1], s5, v2, v[3:4]
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s7
-; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v5, vcc
-; GFX9-NEXT: v_sub_u32_e32 v3, s17, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s6
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x4f800000, v4
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s5, v2, v[4:5]
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s7
+; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v6, vcc
+; GFX9-NEXT: v_sub_u32_e32 v4, s17, v6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s6
+; GFX9-NEXT: v_mul_f32_e32 v5, 0x4f800000, v5
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s5, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1]
-; GFX9-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GFX9-NEXT: v_add_f32_e32 v5, v5, v6
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v5
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s4, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s5, v1
-; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4
-; GFX9-NEXT: v_cndmask_b32_e64 v11, v6, v5, s[0:1]
-; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mul_f32_e32 v6, 0xcf800000, v5
-; GFX9-NEXT: v_add_f32_e32 v4, v6, v4
-; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v4
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v3, v9, vcc
-; GFX9-NEXT: v_cvt_u32_f32_e32 v14, v5
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s2, v12, 0
+; GFX9-NEXT: v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v7, v6, s[0:1]
+; GFX9-NEXT: v_mul_f32_e32 v6, 0x2f800000, v5
+; GFX9-NEXT: v_trunc_f32_e32 v6, v6
+; GFX9-NEXT: v_mul_f32_e32 v7, 0xcf800000, v6
+; GFX9-NEXT: v_add_f32_e32 v5, v7, v5
+; GFX9-NEXT: v_cvt_u32_f32_e32 v12, v5
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v4, v10, vcc
+; GFX9-NEXT: v_cvt_u32_f32_e32 v14, v6
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s2, v12, 0
; GFX9-NEXT: v_subrev_co_u32_e32 v15, vcc, s4, v0
; GFX9-NEXT: v_subbrev_co_u32_e64 v16, s[0:1], 0, v13, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s5, v16
; GFX9-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[0:1]
-; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[0:1], s2, v14, v[4:5]
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s2, v14, v[5:6]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s4, v15
-; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
-; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v13, v9, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[0:1], s3, v12, v[5:6]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, v13, v10, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[0:1], s3, v12, v[6:7]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s5, v16
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v17, v4, s[0:1]
-; GFX9-NEXT: v_mul_lo_u32 v4, v14, v3
-; GFX9-NEXT: v_mul_lo_u32 v5, v12, v7
-; GFX9-NEXT: v_mul_hi_u32 v6, v12, v3
+; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v5, s[0:1]
+; GFX9-NEXT: v_mul_lo_u32 v5, v14, v4
+; GFX9-NEXT: v_mul_lo_u32 v6, v12, v8
+; GFX9-NEXT: v_mul_hi_u32 v7, v12, v4
; GFX9-NEXT: v_add_co_u32_e64 v17, s[0:1], 1, v2
-; GFX9-NEXT: v_addc_co_u32_e64 v18, s[0:1], 0, v10, s[0:1]
-; GFX9-NEXT: v_add_co_u32_e64 v4, s[0:1], v4, v5
+; GFX9-NEXT: v_addc_co_u32_e64 v18, s[0:1], 0, v3, s[0:1]
+; GFX9-NEXT: v_add_co_u32_e64 v5, s[0:1], v5, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[0:1]
+; GFX9-NEXT: v_add_co_u32_e64 v5, s[0:1], v5, v7
+; GFX9-NEXT: v_mul_hi_u32 v4, v14, v4
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
+; GFX9-NEXT: v_mul_lo_u32 v7, v14, v8
+; GFX9-NEXT: v_add_u32_e32 v5, v6, v5
+; GFX9-NEXT: v_mul_hi_u32 v6, v12, v8
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
+; GFX9-NEXT: v_add_co_u32_e64 v4, s[0:1], v7, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[0:1]
; GFX9-NEXT: v_add_co_u32_e64 v4, s[0:1], v4, v6
-; GFX9-NEXT: v_mul_hi_u32 v3, v14, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; GFX9-NEXT: v_mul_lo_u32 v6, v14, v7
-; GFX9-NEXT: v_add_u32_e32 v4, v5, v4
-; GFX9-NEXT: v_mul_hi_u32 v5, v12, v7
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8
-; GFX9-NEXT: v_add_co_u32_e64 v3, s[0:1], v6, v3
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[0:1]
-; GFX9-NEXT: v_add_co_u32_e64 v3, s[0:1], v3, v5
+; GFX9-NEXT: v_add_u32_e32 v6, v7, v6
+; GFX9-NEXT: v_mul_hi_u32 v7, v14, v8
+; GFX9-NEXT: v_add_co_u32_e64 v4, s[0:1], v4, v5
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; GFX9-NEXT: v_add_u32_e32 v5, v6, v5
-; GFX9-NEXT: v_mul_hi_u32 v6, v14, v7
-; GFX9-NEXT: v_add_co_u32_e64 v3, s[0:1], v3, v4
-; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; GFX9-NEXT: v_add3_u32 v4, v5, v4, v6
-; GFX9-NEXT: v_add_co_u32_e64 v12, s[0:1], v12, v3
-; GFX9-NEXT: v_addc_co_u32_e64 v14, s[0:1], v14, v4, s[0:1]
+; GFX9-NEXT: v_add3_u32 v5, v6, v5, v7
+; GFX9-NEXT: v_add_co_u32_e64 v12, s[0:1], v12, v4
+; GFX9-NEXT: v_addc_co_u32_e64 v14, s[0:1], v14, v5, s[0:1]
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s2, v12, 0
-; GFX9-NEXT: v_add_co_u32_e64 v3, s[0:1], 1, v17
+; GFX9-NEXT: v_add_co_u32_e64 v8, s[0:1], 1, v17
; GFX9-NEXT: v_addc_co_u32_e64 v19, s[0:1], 0, v18, s[0:1]
; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[0:1], s2, v14, v[5:6]
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v17, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[0:1], s3, v12, v[6:7]
; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v11
-; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e64 v3, v10, v5, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
; GFX9-NEXT: v_mul_lo_u32 v5, v14, v4
; GFX9-NEXT: v_mul_lo_u32 v6, v12, v8
-; GFX9-NEXT: v_mul_hi_u32 v10, v12, v4
; GFX9-NEXT: v_subrev_co_u32_e64 v7, s[2:3], s4, v15
-; GFX9-NEXT: v_subbrev_co_u32_e64 v9, s[2:3], 0, v13, s[2:3]
+; GFX9-NEXT: v_subbrev_co_u32_e64 v9, s[2:3], 0, v10, s[2:3]
+; GFX9-NEXT: v_mul_hi_u32 v10, v12, v4
; GFX9-NEXT: v_add_co_u32_e64 v5, s[2:3], v5, v6
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[2:3]
; GFX9-NEXT: v_add_co_u32_e64 v5, s[2:3], v5, v10
@@ -1461,7 +1460,7 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mul_lo_u32 v10, s19, v5
; GFX9-NEXT: v_add_u32_e32 v6, v8, v6
; GFX9-NEXT: v_mul_hi_u32 v8, s18, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v13, s[0:1]
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v10, v4
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[2:3]
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v4, v8
@@ -1471,16 +1470,17 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[2:3], s6, v11, 0
; GFX9-NEXT: v_add_u32_e32 v8, v10, v8
; GFX9-NEXT: v_add3_u32 v10, v8, v6, v12
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v16, v9, vcc
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[2:3], s6, v10, v[5:6]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v6, v0, v7, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v7, v1, v13, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s7, v11, v[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, s19
; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, s18, v4
; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v0, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v1
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: v_sub_u32_e32 v0, s19, v0
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v8
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomic-cmpxchg.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomic-cmpxchg.ll
index 4f40948cab0a2..d029dbe08d7d0 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomic-cmpxchg.ll
@@ -376,47 +376,47 @@ define void @flat_atomic_cmpxchg_i64_ret_av_av__av(ptr %ptr) #0 {
; CHECK-LABEL: flat_atomic_cmpxchg_i64_ret_av_av__av:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v0
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[2:3]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[0:1]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB12_2
; CHECK-NEXT: ; %bb.1: ; %atomicrmw.global
; CHECK-NEXT: buffer_wbl2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[6:7], v[0:3] glc
+; CHECK-NEXT: flat_atomic_cmpswap_x2 v[6:7], v[4:5], v[0:3] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_invl2
; CHECK-NEXT: buffer_wbinvl1_vol
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: .LBB12_2: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB12_4
; CHECK-NEXT: ; %bb.3: ; %atomicrmw.private
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
-; CHECK-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; CHECK-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: .LBB12_4: ; %atomicrmw.phi
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; use v[4:5]
+; CHECK-NEXT: ; use v[6:7]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -433,47 +433,47 @@ define void @flat_atomic_cmpxchg_i64_ret_av_av__v(ptr %ptr) #0 {
; CHECK-LABEL: flat_atomic_cmpxchg_i64_ret_av_av__v:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v0
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[2:3]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[0:1]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB13_2
; CHECK-NEXT: ; %bb.1: ; %atomicrmw.global
; CHECK-NEXT: buffer_wbl2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[6:7], v[0:3] glc
+; CHECK-NEXT: flat_atomic_cmpswap_x2 v[6:7], v[4:5], v[0:3] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_invl2
; CHECK-NEXT: buffer_wbinvl1_vol
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: .LBB13_2: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB13_4
; CHECK-NEXT: ; %bb.3: ; %atomicrmw.private
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
-; CHECK-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; CHECK-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: .LBB13_4: ; %atomicrmw.phi
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; use v[4:5]
+; CHECK-NEXT: ; use v[6:7]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -875,47 +875,47 @@ define void @flat_atomic_cmpxchg_i64_ret_av_v__av(ptr %ptr) #0 {
; CHECK-LABEL: flat_atomic_cmpxchg_i64_ret_av_v__av:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v0
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[2:3]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[0:1]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB20_2
; CHECK-NEXT: ; %bb.1: ; %atomicrmw.global
; CHECK-NEXT: buffer_wbl2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[6:7], v[0:3] glc
+; CHECK-NEXT: flat_atomic_cmpswap_x2 v[6:7], v[4:5], v[0:3] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_invl2
; CHECK-NEXT: buffer_wbinvl1_vol
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: .LBB20_2: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB20_4
; CHECK-NEXT: ; %bb.3: ; %atomicrmw.private
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
-; CHECK-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; CHECK-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: .LBB20_4: ; %atomicrmw.phi
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; use v[4:5]
+; CHECK-NEXT: ; use v[6:7]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -932,47 +932,47 @@ define void @flat_atomic_cmpxchg_i64_ret_v_av__av(ptr %ptr) #0 {
; CHECK-LABEL: flat_atomic_cmpxchg_i64_ret_v_av__av:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v0
+; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v0
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
-; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
+; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[2:3]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[0:1]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB21_2
; CHECK-NEXT: ; %bb.1: ; %atomicrmw.global
; CHECK-NEXT: buffer_wbl2
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[6:7], v[0:3] glc
+; CHECK-NEXT: flat_atomic_cmpswap_x2 v[6:7], v[4:5], v[0:3] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_invl2
; CHECK-NEXT: buffer_wbinvl1_vol
-; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7
+; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: .LBB21_2: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB21_4
; CHECK-NEXT: ; %bb.3: ; %atomicrmw.private
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; CHECK-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc
-; CHECK-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
+; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; CHECK-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; CHECK-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; CHECK-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; CHECK-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; CHECK-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; CHECK-NEXT: .LBB21_4: ; %atomicrmw.phi
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; use v[4:5]
+; CHECK-NEXT: ; use v[6:7]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/add_i1.ll b/llvm/test/CodeGen/AMDGPU/add_i1.ll
index ac5386bdfc06f..c718e22ee687d 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i1.ll
@@ -111,25 +111,25 @@ define amdgpu_kernel void @add_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a,
; GFX9-LABEL: add_i1_cf:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX9-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[8:9]
; GFX9-NEXT: s_cbranch_execz .LBB2_2
; GFX9-NEXT: ; %bb.1: ; %else
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT: global_load_ubyte v0, v0, s[6:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0
; GFX9-NEXT: .LBB2_2: ; %Flow
-; GFX9-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_andn2_saveexec_b64 s[6:7], s[8:9]
; GFX9-NEXT: s_cbranch_execz .LBB2_4
; GFX9-NEXT: ; %bb.3: ; %if
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_andn2_b64 s[2:3], s[4:5], exec
@@ -139,7 +139,6 @@ define amdgpu_kernel void @add_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a,
; GFX9-NEXT: s_or_b64 s[4:5], s[2:3], s[4:5]
; GFX9-NEXT: .LBB2_4: ; %endif
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b64 s[2:3], s[4:5], -1
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index b1133f6050334..f73b91be9cda2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -26073,109 +26073,113 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v63, s30, 0
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: v_writelane_b32 v63, s34, 2
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_and_b32 s30, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s31, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
; SI-NEXT: v_writelane_b32 v63, s35, 3
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_and_b32 s34, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s35, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s36, 4
+; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s36
+; SI-NEXT: v_readfirstlane_b32 s92, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s34
; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s30
; SI-NEXT: v_readfirstlane_b32 s88, v6
; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s93
; SI-NEXT: v_readfirstlane_b32 s78, v7
; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s91
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s89
; SI-NEXT: v_readfirstlane_b32 s74, v9
; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
; SI-NEXT: v_readfirstlane_b32 s72, v10
; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
; SI-NEXT: v_readfirstlane_b32 s60, v12
; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
; SI-NEXT: v_readfirstlane_b32 s58, v13
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
; SI-NEXT: v_readfirstlane_b32 s56, v14
; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
; SI-NEXT: v_readfirstlane_b32 s46, v15
; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s44, v16
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s88, s88, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s47
-; SI-NEXT: v_writelane_b32 v63, s36, 4
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s76, s76, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: v_readfirstlane_b32 s42, v17
+; SI-NEXT: v_readfirstlane_b32 s44, v16
+; SI-NEXT: v_readfirstlane_b32 s94, v3
; SI-NEXT: v_writelane_b32 v63, s37, 5
; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s28, 16
; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s27, 16
; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s26, 16
+; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_lshl_b32 s13, s25, 16
; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
; SI-NEXT: s_lshl_b32 s15, s24, 16
@@ -26189,34 +26193,32 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s19, s19, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
; SI-NEXT: s_lshl_b32 s42, s42, 16
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_lshl_b32 s58, s58, 16
; SI-NEXT: s_lshl_b32 s60, s60, 16
; SI-NEXT: s_lshl_b32 s62, s62, 16
; SI-NEXT: s_lshl_b32 s72, s72, 16
; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_lshl_b32 s88, s88, 16
; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
; SI-NEXT: s_lshl_b32 s92, s92, 16
+; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s37, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -26233,446 +26235,452 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s29
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v47, 1.0, s27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s36
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s93
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s23
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v58, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s37
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v43, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s42
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v54, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s44
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s42
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB19_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v55
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v53
+; SI-NEXT: v_mov_b32_e32 v0, v34
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v44, v1
-; SI-NEXT: v_mov_b32_e32 v43, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[34:35], 16
+; SI-NEXT: v_mov_b32_e32 v1, v48
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v37
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v37, v36
-; SI-NEXT: v_mov_b32_e32 v36, v47
-; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v4
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v37, v51
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v10
+; SI-NEXT: v_mov_b32_e32 v15, v47
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v13
+; SI-NEXT: v_mov_b32_e32 v19, v44
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v18
+; SI-NEXT: v_mov_b32_e32 v51, v58
+; SI-NEXT: v_mov_b32_e32 v53, v36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v2, v34
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_mov_b32_e32 v3, v17
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v4, v20
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v50, v2
+; SI-NEXT: v_mov_b32_e32 v2, v33
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v5, v19
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v6
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[33:34], 16
+; SI-NEXT: v_mov_b32_e32 v3, v61
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v6, v13
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[3:4], v[61:62], 16
+; SI-NEXT: v_mov_b32_e32 v41, v40
+; SI-NEXT: v_lshr_b64 v[4:5], v[37:38], 16
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[39:40], 16
+; SI-NEXT: v_mov_b32_e32 v6, v60
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v56
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[13:14], 16
-; SI-NEXT: v_mov_b32_e32 v7, v31
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[60:61], 16
+; SI-NEXT: v_mov_b32_e32 v7, v21
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[31:32], 16
+; SI-NEXT: v_lshr_b64 v[7:8], v[21:22], 16
; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v44, v45
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; SI-NEXT: v_mov_b32_e32 v9, v25
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v10, v30
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v9, v26
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[9:10], v[26:27], 16
+; SI-NEXT: v_mov_b32_e32 v10, v25
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[25:26], 16
+; SI-NEXT: v_mov_b32_e32 v11, v46
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[11:12], v[46:47], 16
+; SI-NEXT: v_mov_b32_e32 v12, v28
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[12:13], v[28:29], 16
+; SI-NEXT: v_mov_b32_e32 v13, v55
+; SI-NEXT: v_mov_b32_e32 v47, v15
+; SI-NEXT: v_mov_b32_e32 v15, v56
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v11, v27
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[13:14], v[55:56], 16
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v56, v15
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_mov_b32_e32 v38, v39
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; SI-NEXT: v_mov_b32_e32 v14, v30
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
-; SI-NEXT: v_mov_b32_e32 v12, v61
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[12:13], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v13, v29
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v14, v15
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v62, v39
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v21
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[14:15], v[30:31], 16
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v24
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_mov_b32_e32 v15, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[15:16], v[16:17], 16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; SI-NEXT: v_mov_b32_e32 v16, v32
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
+; SI-NEXT: v_lshr_b64 v[16:17], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v17, v43
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[17:18], v[43:44], 16
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v44, v19
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v18
+; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
-; SI-NEXT: v_mov_b32_e32 v15, v60
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; SI-NEXT: v_mov_b32_e32 v19, v20
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v16, v33
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[16:17], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v17, v57
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[57:58], 16
+; SI-NEXT: v_mov_b32_e32 v22, v31
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_mov_b32_e32 v58, v51
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v51
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[45:46], 16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[57:58], 16
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v47
-; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
-; SI-NEXT: v_lshr_b64 v[19:20], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v20, v38
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[38:39], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
-; SI-NEXT: v_mov_b32_e32 v39, v62
-; SI-NEXT: v_mov_b32_e32 v58, v56
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_mov_b32_e32 v21, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_lshr_b64 v[61:62], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v51, v37
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v24
+; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
-; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[54:55], 16
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_mov_b32_e32 v39, v32
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[29:30], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v36, v53
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v47, v36
-; SI-NEXT: v_mov_b32_e32 v36, v37
-; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
-; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[52:53], 16
; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v41, v40
; SI-NEXT: s_branch .LBB19_3
; SI-NEXT: .LBB19_2:
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v58, v56
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB19_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v37, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v33, v42
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_vccnz .LBB19_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_mov_b32_e32 v34, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v36
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -26681,32 +26689,24 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
@@ -26715,9 +26715,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
@@ -26726,9 +26726,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
@@ -26737,71 +26737,69 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v33
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v35
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -26810,15 +26808,25 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v58
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
@@ -26827,42 +26835,58 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v39
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v54
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v42
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v51
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v41
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v54
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
@@ -26877,6 +26901,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -26889,7 +26914,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -62005,109 +62030,113 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v63, s30, 0
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: v_writelane_b32 v63, s34, 2
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_and_b32 s30, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s31, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
; SI-NEXT: v_writelane_b32 v63, s35, 3
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_and_b32 s34, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s35, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s36, 4
+; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s36
+; SI-NEXT: v_readfirstlane_b32 s92, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s34
; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s30
; SI-NEXT: v_readfirstlane_b32 s88, v6
; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s93
; SI-NEXT: v_readfirstlane_b32 s78, v7
; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s91
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s89
; SI-NEXT: v_readfirstlane_b32 s74, v9
; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
; SI-NEXT: v_readfirstlane_b32 s72, v10
; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
; SI-NEXT: v_readfirstlane_b32 s60, v12
; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
; SI-NEXT: v_readfirstlane_b32 s58, v13
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
; SI-NEXT: v_readfirstlane_b32 s56, v14
; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
; SI-NEXT: v_readfirstlane_b32 s46, v15
; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s44, v16
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s88, s88, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s47
-; SI-NEXT: v_writelane_b32 v63, s36, 4
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s76, s76, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: v_readfirstlane_b32 s42, v17
+; SI-NEXT: v_readfirstlane_b32 s44, v16
+; SI-NEXT: v_readfirstlane_b32 s94, v3
; SI-NEXT: v_writelane_b32 v63, s37, 5
; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s28, 16
; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s27, 16
; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s26, 16
+; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_lshl_b32 s13, s25, 16
; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
; SI-NEXT: s_lshl_b32 s15, s24, 16
@@ -62121,34 +62150,32 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s19, s19, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
; SI-NEXT: s_lshl_b32 s42, s42, 16
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_lshl_b32 s58, s58, 16
; SI-NEXT: s_lshl_b32 s60, s60, 16
; SI-NEXT: s_lshl_b32 s62, s62, 16
; SI-NEXT: s_lshl_b32 s72, s72, 16
; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_lshl_b32 s88, s88, 16
; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
; SI-NEXT: s_lshl_b32 s92, s92, 16
+; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s37, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -62165,446 +62192,452 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s29
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v47, 1.0, s27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s36
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s93
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s23
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v58, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s37
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v43, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s42
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v54, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s44
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s42
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB43_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v55
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v53
+; SI-NEXT: v_mov_b32_e32 v0, v34
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v44, v1
-; SI-NEXT: v_mov_b32_e32 v43, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[34:35], 16
+; SI-NEXT: v_mov_b32_e32 v1, v48
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v37
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v37, v36
-; SI-NEXT: v_mov_b32_e32 v36, v47
-; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v4
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v37, v51
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v10
+; SI-NEXT: v_mov_b32_e32 v15, v47
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v13
+; SI-NEXT: v_mov_b32_e32 v19, v44
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v18
+; SI-NEXT: v_mov_b32_e32 v51, v58
+; SI-NEXT: v_mov_b32_e32 v53, v36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v2, v34
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_mov_b32_e32 v3, v17
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v4, v20
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v50, v2
+; SI-NEXT: v_mov_b32_e32 v2, v33
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v5, v19
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v6
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[33:34], 16
+; SI-NEXT: v_mov_b32_e32 v3, v61
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v6, v13
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[3:4], v[61:62], 16
+; SI-NEXT: v_mov_b32_e32 v41, v40
+; SI-NEXT: v_lshr_b64 v[4:5], v[37:38], 16
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[39:40], 16
+; SI-NEXT: v_mov_b32_e32 v6, v60
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v56
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[13:14], 16
-; SI-NEXT: v_mov_b32_e32 v7, v31
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[60:61], 16
+; SI-NEXT: v_mov_b32_e32 v7, v21
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[31:32], 16
+; SI-NEXT: v_lshr_b64 v[7:8], v[21:22], 16
; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v44, v45
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; SI-NEXT: v_mov_b32_e32 v9, v25
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v10, v30
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v9, v26
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[9:10], v[26:27], 16
+; SI-NEXT: v_mov_b32_e32 v10, v25
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[25:26], 16
+; SI-NEXT: v_mov_b32_e32 v11, v46
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[11:12], v[46:47], 16
+; SI-NEXT: v_mov_b32_e32 v12, v28
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[12:13], v[28:29], 16
+; SI-NEXT: v_mov_b32_e32 v13, v55
+; SI-NEXT: v_mov_b32_e32 v47, v15
+; SI-NEXT: v_mov_b32_e32 v15, v56
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v11, v27
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[13:14], v[55:56], 16
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v56, v15
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_mov_b32_e32 v38, v39
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; SI-NEXT: v_mov_b32_e32 v14, v30
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
-; SI-NEXT: v_mov_b32_e32 v12, v61
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[12:13], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v13, v29
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v14, v15
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v62, v39
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v21
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[14:15], v[30:31], 16
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v24
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_mov_b32_e32 v15, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[15:16], v[16:17], 16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; SI-NEXT: v_mov_b32_e32 v16, v32
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
+; SI-NEXT: v_lshr_b64 v[16:17], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v17, v43
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[17:18], v[43:44], 16
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v44, v19
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v18
+; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
-; SI-NEXT: v_mov_b32_e32 v15, v60
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; SI-NEXT: v_mov_b32_e32 v19, v20
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v16, v33
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[16:17], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v17, v57
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[57:58], 16
+; SI-NEXT: v_mov_b32_e32 v22, v31
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_mov_b32_e32 v58, v51
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v51
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[45:46], 16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[57:58], 16
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v47
-; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
-; SI-NEXT: v_lshr_b64 v[19:20], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v20, v38
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[38:39], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
-; SI-NEXT: v_mov_b32_e32 v39, v62
-; SI-NEXT: v_mov_b32_e32 v58, v56
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_mov_b32_e32 v21, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_lshr_b64 v[61:62], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v51, v37
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v24
+; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
-; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[54:55], 16
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_mov_b32_e32 v39, v32
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[29:30], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v36, v53
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v47, v36
-; SI-NEXT: v_mov_b32_e32 v36, v37
-; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
-; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[52:53], 16
; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v41, v40
; SI-NEXT: s_branch .LBB43_3
; SI-NEXT: .LBB43_2:
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v58, v56
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB43_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v37, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v33, v42
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_vccnz .LBB43_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_mov_b32_e32 v34, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v36
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -62613,32 +62646,24 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
@@ -62647,9 +62672,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
@@ -62658,9 +62683,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
@@ -62669,71 +62694,69 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v33
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v35
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -62742,15 +62765,25 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v58
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
@@ -62759,42 +62792,58 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v39
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v54
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v42
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v51
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v41
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v54
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
@@ -62809,6 +62858,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -62821,7 +62871,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -95535,109 +95585,113 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v63, s30, 0
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: v_writelane_b32 v63, s34, 2
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_and_b32 s30, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s31, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
; SI-NEXT: v_writelane_b32 v63, s35, 3
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_and_b32 s34, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s35, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s36, 4
+; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s36
+; SI-NEXT: v_readfirstlane_b32 s92, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s34
; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s30
; SI-NEXT: v_readfirstlane_b32 s88, v6
; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s93
; SI-NEXT: v_readfirstlane_b32 s78, v7
; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s91
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s89
; SI-NEXT: v_readfirstlane_b32 s74, v9
; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
; SI-NEXT: v_readfirstlane_b32 s72, v10
; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
; SI-NEXT: v_readfirstlane_b32 s60, v12
; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
; SI-NEXT: v_readfirstlane_b32 s58, v13
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
; SI-NEXT: v_readfirstlane_b32 s56, v14
; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
; SI-NEXT: v_readfirstlane_b32 s46, v15
; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s44, v16
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s88, s88, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s47
-; SI-NEXT: v_writelane_b32 v63, s36, 4
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s76, s76, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: v_readfirstlane_b32 s42, v17
+; SI-NEXT: v_readfirstlane_b32 s44, v16
+; SI-NEXT: v_readfirstlane_b32 s94, v3
; SI-NEXT: v_writelane_b32 v63, s37, 5
; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s28, 16
; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s27, 16
; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s26, 16
+; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_lshl_b32 s13, s25, 16
; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
; SI-NEXT: s_lshl_b32 s15, s24, 16
@@ -95651,34 +95705,32 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s19, s19, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
; SI-NEXT: s_lshl_b32 s42, s42, 16
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_lshl_b32 s58, s58, 16
; SI-NEXT: s_lshl_b32 s60, s60, 16
; SI-NEXT: s_lshl_b32 s62, s62, 16
; SI-NEXT: s_lshl_b32 s72, s72, 16
; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_lshl_b32 s88, s88, 16
; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
; SI-NEXT: s_lshl_b32 s92, s92, 16
+; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s37, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -95695,446 +95747,452 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s29
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v47, 1.0, s27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s36
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s93
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s23
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v58, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s37
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v43, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s42
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v54, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s44
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s42
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB63_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v55
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v53
+; SI-NEXT: v_mov_b32_e32 v0, v34
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v44, v1
-; SI-NEXT: v_mov_b32_e32 v43, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[34:35], 16
+; SI-NEXT: v_mov_b32_e32 v1, v48
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v37
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v37, v36
-; SI-NEXT: v_mov_b32_e32 v36, v47
-; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v4
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v37, v51
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v10
+; SI-NEXT: v_mov_b32_e32 v15, v47
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v13
+; SI-NEXT: v_mov_b32_e32 v19, v44
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v18
+; SI-NEXT: v_mov_b32_e32 v51, v58
+; SI-NEXT: v_mov_b32_e32 v53, v36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v2, v34
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_mov_b32_e32 v3, v17
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v4, v20
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v50, v2
+; SI-NEXT: v_mov_b32_e32 v2, v33
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v5, v19
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v6
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[33:34], 16
+; SI-NEXT: v_mov_b32_e32 v3, v61
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v6, v13
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[3:4], v[61:62], 16
+; SI-NEXT: v_mov_b32_e32 v41, v40
+; SI-NEXT: v_lshr_b64 v[4:5], v[37:38], 16
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[39:40], 16
+; SI-NEXT: v_mov_b32_e32 v6, v60
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v56
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[13:14], 16
-; SI-NEXT: v_mov_b32_e32 v7, v31
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[60:61], 16
+; SI-NEXT: v_mov_b32_e32 v7, v21
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[31:32], 16
+; SI-NEXT: v_lshr_b64 v[7:8], v[21:22], 16
; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v44, v45
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; SI-NEXT: v_mov_b32_e32 v9, v25
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v10, v30
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v9, v26
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[9:10], v[26:27], 16
+; SI-NEXT: v_mov_b32_e32 v10, v25
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[25:26], 16
+; SI-NEXT: v_mov_b32_e32 v11, v46
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[11:12], v[46:47], 16
+; SI-NEXT: v_mov_b32_e32 v12, v28
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[12:13], v[28:29], 16
+; SI-NEXT: v_mov_b32_e32 v13, v55
+; SI-NEXT: v_mov_b32_e32 v47, v15
+; SI-NEXT: v_mov_b32_e32 v15, v56
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v11, v27
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[13:14], v[55:56], 16
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v56, v15
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_mov_b32_e32 v38, v39
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; SI-NEXT: v_mov_b32_e32 v14, v30
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
-; SI-NEXT: v_mov_b32_e32 v12, v61
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[12:13], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v13, v29
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v14, v15
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v62, v39
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v21
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[14:15], v[30:31], 16
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v24
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_mov_b32_e32 v15, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[15:16], v[16:17], 16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; SI-NEXT: v_mov_b32_e32 v16, v32
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
+; SI-NEXT: v_lshr_b64 v[16:17], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v17, v43
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[17:18], v[43:44], 16
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v44, v19
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v18
+; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
-; SI-NEXT: v_mov_b32_e32 v15, v60
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; SI-NEXT: v_mov_b32_e32 v19, v20
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v16, v33
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[16:17], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v17, v57
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[57:58], 16
+; SI-NEXT: v_mov_b32_e32 v22, v31
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_mov_b32_e32 v58, v51
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v51
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[45:46], 16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[57:58], 16
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v47
-; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
-; SI-NEXT: v_lshr_b64 v[19:20], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v20, v38
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[38:39], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
-; SI-NEXT: v_mov_b32_e32 v39, v62
-; SI-NEXT: v_mov_b32_e32 v58, v56
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_mov_b32_e32 v21, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_lshr_b64 v[61:62], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v51, v37
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v24
+; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
-; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[54:55], 16
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_mov_b32_e32 v39, v32
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[29:30], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v36, v53
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v47, v36
-; SI-NEXT: v_mov_b32_e32 v36, v37
-; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
-; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[52:53], 16
; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v41, v40
; SI-NEXT: s_branch .LBB63_3
; SI-NEXT: .LBB63_2:
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v58, v56
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB63_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v37, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v33, v42
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_vccnz .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_mov_b32_e32 v34, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v36
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -96143,32 +96201,24 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
@@ -96177,9 +96227,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
@@ -96188,9 +96238,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
@@ -96199,71 +96249,69 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v33
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v35
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -96272,15 +96320,25 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v58
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
@@ -96289,42 +96347,58 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v39
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v54
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v42
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v51
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v41
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v54
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
@@ -96339,6 +96413,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -96351,7 +96426,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -128450,109 +128525,113 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_writelane_b32 v63, s30, 0
; SI-NEXT: v_writelane_b32 v63, s31, 1
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: v_writelane_b32 v63, s34, 2
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_and_b32 s30, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s31, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v1
; SI-NEXT: v_writelane_b32 v63, s35, 3
-; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_and_b32 s34, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s35, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v0
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s12
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
-; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_writelane_b32 v63, s36, 4
+; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s36
+; SI-NEXT: v_readfirstlane_b32 s92, v4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s34
; SI-NEXT: v_readfirstlane_b32 s90, v5
-; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s30
; SI-NEXT: v_readfirstlane_b32 s88, v6
; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s93
; SI-NEXT: v_readfirstlane_b32 s78, v7
; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s91
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s89
; SI-NEXT: v_readfirstlane_b32 s74, v9
; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
; SI-NEXT: v_readfirstlane_b32 s72, v10
; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s75
; SI-NEXT: v_readfirstlane_b32 s60, v12
; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s73
; SI-NEXT: v_readfirstlane_b32 s58, v13
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
; SI-NEXT: v_readfirstlane_b32 s56, v14
; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s61
; SI-NEXT: v_readfirstlane_b32 s46, v15
; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s59
-; SI-NEXT: v_readfirstlane_b32 s44, v16
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s57
-; SI-NEXT: v_readfirstlane_b32 s42, v17
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s88, s88, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s47
-; SI-NEXT: v_writelane_b32 v63, s36, 4
-; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s92, v4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_lshl_b32 s76, s76, 16
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: s_lshl_b32 s56, s56, 16
+; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: v_readfirstlane_b32 s42, v17
+; SI-NEXT: v_readfirstlane_b32 s44, v16
+; SI-NEXT: v_readfirstlane_b32 s94, v3
; SI-NEXT: v_writelane_b32 v63, s37, 5
; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
; SI-NEXT: s_lshl_b32 s7, s28, 16
; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
; SI-NEXT: s_lshl_b32 s9, s27, 16
; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
; SI-NEXT: s_lshl_b32 s11, s26, 16
+; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
; SI-NEXT: s_lshl_b32 s13, s25, 16
; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
; SI-NEXT: s_lshl_b32 s15, s24, 16
@@ -128566,34 +128645,32 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s19, s19, 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s16, s16, 16
+; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
; SI-NEXT: s_lshl_b32 s42, s42, 16
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
; SI-NEXT: s_lshl_b32 s44, s44, 16
-; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_lshl_b32 s58, s58, 16
; SI-NEXT: s_lshl_b32 s60, s60, 16
; SI-NEXT: s_lshl_b32 s62, s62, 16
; SI-NEXT: s_lshl_b32 s72, s72, 16
; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_lshl_b32 s88, s88, 16
; SI-NEXT: s_lshl_b32 s90, s90, 16
-; SI-NEXT: s_and_b32 s93, s92, 0xffff0000
; SI-NEXT: s_lshl_b32 s92, s92, 16
+; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s94, 16
-; SI-NEXT: s_and_b32 s36, vcc_lo, 0xffff0000
; SI-NEXT: s_lshl_b32 s37, vcc_lo, 16
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v18
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -128610,446 +128687,452 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s29
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v5, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s28
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v47, 1.0, s27
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s46
; SI-NEXT: v_mul_f32_e64 v6, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v8, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v11, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s36
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s93
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v43, 1.0, s16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s23
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s45
+; SI-NEXT: v_mul_f32_e64 v58, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s23
; SI-NEXT: v_mul_f32_e64 v23, 1.0, s15
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s37
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s35
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s31
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v43, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v59, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s56
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s44
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s42
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v54, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s44
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s42
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB79_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v55
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v53
+; SI-NEXT: v_mov_b32_e32 v0, v34
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v50
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v44, v1
-; SI-NEXT: v_mov_b32_e32 v43, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[0:1], v[34:35], 16
+; SI-NEXT: v_mov_b32_e32 v1, v48
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v37
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v37, v36
-; SI-NEXT: v_mov_b32_e32 v36, v47
-; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v4
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v39, v38
+; SI-NEXT: v_mov_b32_e32 v37, v51
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v47
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v10
+; SI-NEXT: v_mov_b32_e32 v15, v47
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v13
+; SI-NEXT: v_mov_b32_e32 v19, v44
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v18
+; SI-NEXT: v_mov_b32_e32 v51, v58
+; SI-NEXT: v_mov_b32_e32 v53, v36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[43:44], 16
-; SI-NEXT: v_mov_b32_e32 v2, v34
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[2:3], v[34:35], 16
-; SI-NEXT: v_mov_b32_e32 v3, v17
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[17:18], 16
-; SI-NEXT: v_mov_b32_e32 v4, v20
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v50, v2
+; SI-NEXT: v_mov_b32_e32 v2, v33
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[4:5], v[20:21], 16
-; SI-NEXT: v_mov_b32_e32 v5, v19
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v6
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[33:34], 16
+; SI-NEXT: v_mov_b32_e32 v3, v61
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[5:6], v[19:20], 16
-; SI-NEXT: v_mov_b32_e32 v6, v13
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[3:4], v[61:62], 16
+; SI-NEXT: v_mov_b32_e32 v41, v40
+; SI-NEXT: v_lshr_b64 v[4:5], v[37:38], 16
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[39:40], 16
+; SI-NEXT: v_mov_b32_e32 v6, v60
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v56
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[6:7], v[13:14], 16
-; SI-NEXT: v_mov_b32_e32 v7, v31
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[60:61], 16
+; SI-NEXT: v_mov_b32_e32 v7, v21
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[7:8], v[31:32], 16
+; SI-NEXT: v_lshr_b64 v[7:8], v[21:22], 16
; SI-NEXT: v_mov_b32_e32 v8, v23
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v44, v45
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
-; SI-NEXT: v_mov_b32_e32 v9, v25
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v10, v30
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v9, v26
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[9:10], v[26:27], 16
+; SI-NEXT: v_mov_b32_e32 v10, v25
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v11
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[25:26], 16
+; SI-NEXT: v_mov_b32_e32 v11, v46
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[11:12], v[46:47], 16
+; SI-NEXT: v_mov_b32_e32 v12, v28
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[12:13], v[28:29], 16
+; SI-NEXT: v_mov_b32_e32 v13, v55
+; SI-NEXT: v_mov_b32_e32 v47, v15
+; SI-NEXT: v_mov_b32_e32 v15, v56
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[10:11], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v11, v27
-; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[13:14], v[55:56], 16
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v56, v15
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v40, v41
+; SI-NEXT: v_mov_b32_e32 v38, v39
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; SI-NEXT: v_mov_b32_e32 v14, v30
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
-; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
-; SI-NEXT: v_mov_b32_e32 v12, v61
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[12:13], v[61:62], 16
-; SI-NEXT: v_mov_b32_e32 v13, v29
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[29:30], 16
-; SI-NEXT: v_mov_b32_e32 v14, v15
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v62, v39
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v62
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v21
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[14:15], v[30:31], 16
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v24
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; SI-NEXT: v_mov_b32_e32 v15, v16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[15:16], v[16:17], 16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; SI-NEXT: v_mov_b32_e32 v16, v32
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
+; SI-NEXT: v_lshr_b64 v[16:17], v[32:33], 16
+; SI-NEXT: v_mov_b32_e32 v17, v43
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[17:18], v[43:44], 16
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v44, v19
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v22
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v18
+; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
-; SI-NEXT: v_mov_b32_e32 v15, v60
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; SI-NEXT: v_mov_b32_e32 v19, v20
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[60:61], 16
-; SI-NEXT: v_mov_b32_e32 v16, v33
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[19:20], v[20:21], 16
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[16:17], v[33:34], 16
-; SI-NEXT: v_mov_b32_e32 v17, v57
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[57:58], 16
+; SI-NEXT: v_mov_b32_e32 v22, v31
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v26
+; SI-NEXT: v_mov_b32_e32 v58, v51
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v51
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
+; SI-NEXT: v_lshr_b64 v[23:24], v[45:46], 16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[57:58], 16
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v47
-; SI-NEXT: v_lshr_b64 v[18:19], v[59:60], 16
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v20
-; SI-NEXT: v_lshr_b64 v[19:20], v[56:57], 16
-; SI-NEXT: v_mov_b32_e32 v20, v38
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[20:21], v[38:39], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
-; SI-NEXT: v_mov_b32_e32 v39, v62
-; SI-NEXT: v_mov_b32_e32 v58, v56
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: v_mov_b32_e32 v21, v22
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v28
+; SI-NEXT: v_lshr_b64 v[61:62], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v51, v37
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v24
+; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
-; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
-; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
-; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
-; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
-; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[26:27], v[54:55], 16
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_mov_b32_e32 v39, v32
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_lshr_b64 v[28:29], v[35:36], 16
+; SI-NEXT: v_lshr_b64 v[29:30], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v36, v53
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v47, v36
-; SI-NEXT: v_mov_b32_e32 v36, v37
-; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
-; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[30:31], v[52:53], 16
; SI-NEXT: v_mov_b32_e32 v31, v61
-; SI-NEXT: v_mov_b32_e32 v41, v40
; SI-NEXT: s_branch .LBB79_3
; SI-NEXT: .LBB79_2:
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v58, v56
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB79_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v37, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v35, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v33, v42
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_vccnz .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-NEXT: v_mov_b32_e32 v34, v42
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v36
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
@@ -129058,32 +129141,24 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v51
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v56
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
@@ -129092,9 +129167,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
@@ -129103,9 +129178,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
@@ -129114,71 +129189,69 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v33
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v35
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
-; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -129187,15 +129260,25 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v59
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
; SI-NEXT: v_lshr_b64 v[18:19], v[18:19], 16
-; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v58
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v19
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
@@ -129204,42 +129287,58 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v57
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
+; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
; SI-NEXT: v_lshr_b64 v[22:23], v[22:23], 16
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v46
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshr_b64 v[23:24], v[23:24], 16
-; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
; SI-NEXT: v_lshr_b64 v[24:25], v[24:25], 16
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v49
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v39
; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshr_b64 v[25:26], v[25:26], 16
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v45
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v54
; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshr_b64 v[26:27], v[26:27], 16
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v42
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v41
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v51
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v41
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
; SI-NEXT: v_lshr_b64 v[29:30], v[29:30], 16
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v54
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v52
; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshr_b64 v[30:31], v[30:31], 16
; SI-NEXT: v_mov_b32_e32 v31, v32
@@ -129254,6 +129353,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -129266,7 +129366,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -142937,81 +143037,81 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:372
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:364
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:356
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:340
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:324
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:276
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
@@ -143019,55 +143119,55 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v72.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v180.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v163.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
@@ -143089,13 +143189,13 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
@@ -143109,231 +143209,231 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB88_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v163, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v57, v41, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v72, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v178, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v44, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v46, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v62, v47, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v48.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v48.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v74, v58, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v97, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v78, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v99, v85, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v116, v100, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v86, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v128, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v130, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v145, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v98, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v161, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v160, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v133, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v165, v146, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v167, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v118, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v45, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v63, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v176, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v135, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v119, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v166, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v179, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v36.h
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v73, v61, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v39.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v182, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v56, v183, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v59, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v75, v60, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
@@ -143354,61 +143454,59 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB88_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v75.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v56.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
@@ -143420,88 +143518,88 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v118.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v102.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v133.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v128.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v103.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v100.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v96.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v87.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v97.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
@@ -143510,7 +143608,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
@@ -143519,11 +143617,11 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v77.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
@@ -143531,120 +143629,120 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v78.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v74.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v58.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v62.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v73.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v61.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v63.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v57.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v180.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v44.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v40.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v166.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v162.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v164.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v147.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v135.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v144.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v132.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v117.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v112.l
; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
@@ -169422,81 +169520,81 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:372
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:364
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:356
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:340
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:324
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:276
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
@@ -169504,55 +169602,55 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v72.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v180.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v163.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
@@ -169574,13 +169672,13 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
@@ -169594,231 +169692,231 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB92_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v163, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v57, v41, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v72, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v178, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v44, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v46, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v62, v47, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v48.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v48.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v74, v58, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v97, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v78, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v99, v85, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v116, v100, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v86, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v128, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v130, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v145, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v98, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v161, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v160, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v133, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v165, v146, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v167, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v118, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v45, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v63, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v176, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v135, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v119, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v166, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v179, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v36.h
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v73, v61, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v39.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v182, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v56, v183, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v59, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v75, v60, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
@@ -169839,61 +169937,59 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB92_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v75.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v56.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
@@ -169905,88 +170001,88 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v118.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v102.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v133.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v128.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v103.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v100.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v96.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v87.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v97.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
@@ -169995,7 +170091,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
@@ -170004,11 +170100,11 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v77.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
@@ -170016,120 +170112,120 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v78.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v74.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v58.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v62.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v73.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v61.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v63.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v57.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v180.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v44.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v40.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v166.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v162.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v164.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v147.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v135.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v144.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v132.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v117.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v112.l
; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
@@ -190049,81 +190145,81 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:380
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:376
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:372
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:364
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:356
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:348
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:344
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:340
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:324
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:316
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:276
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:240
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:224
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:208
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:204
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:252
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:48
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
@@ -190131,55 +190227,55 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v79, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v73, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v77, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v76, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v56, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v63, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v41, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v43, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v72.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v181.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v180.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v163.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
@@ -190201,13 +190297,13 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
@@ -190221,231 +190317,231 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v130, v97, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v165, v135, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v45, v178, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v163, v117, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v164, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v57, v41, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v60, v41, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v160, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v72, v43, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v178, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v177, v132, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v44, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v51.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v40, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v46, v181, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v58, v42, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v62, v47, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v38.l
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v48.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v50.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v49.l
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v48.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v51.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v50.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v65.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v63, v56, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v98, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v74, v58, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v97, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v54.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v53.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v77, v73, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v102, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v78, v76, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v99, v85, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v55.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v76, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v119, v112, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v54.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v79, v77, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v116, v100, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v64.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v64.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v83, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v116, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v86, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v128, v103, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v96, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v149, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v130, v113, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v99, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v163, v131, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v145, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v103, v86, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v179, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v117, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v98, v83, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v161, v129, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v160, v134, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v148, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v44, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v133, v101, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v165, v146, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v34.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v146, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v167, v144, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v118, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v181, v162, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v118, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v45, v180, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v161, v134, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v57, v182, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v131, v114, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v63, v42, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v43, v167, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v113, v87, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v176, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v135, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v180, v151, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v119, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v147, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v166, v147, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v25.l, v67.h
-; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v164, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v179, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v27.h, v68.h
-; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v36.h
; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v35.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v62, v47, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v73, v61, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v29.h, v69.l
-; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.h, 0xff, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v39.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v30.l, v71.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v59, v183, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v182, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.h, v68.l
; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v67.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v72, v61, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v56, v183, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.h, v69.h
; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v70.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v74, v46, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v59, v177, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.h
; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v31.l, v71.l
; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.h, v70.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v78, v75, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v75, v60, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v52.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v52.h
; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v66.l
; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v66.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v32.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
@@ -190466,61 +190562,59 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v78.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v48.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v39.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v75.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v60.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v74.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v43.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v176.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v71.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v71.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v70.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v46.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v177.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v35.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v69.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v56.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v69.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v70.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v59.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v182.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v68.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v68.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v183.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v151.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.l, v1.l
@@ -190532,88 +190626,88 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v67.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v180.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v150.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v44.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v165.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v66.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v151.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v119.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v66.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v176.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v146.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v167.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v149.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v161.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v161.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v131.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v150.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v134.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v145.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v118.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v131.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v115.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v118.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v149.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v102.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v130.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v129.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v148.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v113.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v133.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v144.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v128.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v101.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v116.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v103.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v103.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v119.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v116.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v86.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v99.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v100.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v96.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v102.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v96.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v87.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v85.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v98.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v97.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v83.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v86.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v81.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v82.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v80.l
@@ -190622,7 +190716,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.h, v1.l
@@ -190631,11 +190725,11 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v79.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v51.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v64.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v64.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v77.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v50.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v0.l
@@ -190643,120 +190737,120 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v65.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v77.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v78.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v48.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v73.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v76.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v48.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v55.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v63.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v74.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v37.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v53.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v58.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v53.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v58.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v62.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.l, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v60.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v47.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v72.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v62.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v73.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v43.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v47.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v57.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v61.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v63.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v40.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v46.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v182.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v42.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v166.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v45.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v181.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v57.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v178.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v164.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v179.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v181.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v45.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v148.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v180.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v177.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v44.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v165.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v40.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v147.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v166.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v162.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v135.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v164.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v147.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v167.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v160.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v113.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v178.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v135.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v130.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v117.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v163.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.h, v160.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v114.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v144.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v101.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.l, 8, v132.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v97.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v117.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v112.l
; GFX11-TRUE16-NEXT: v_and_b16 v32.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v134.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v32.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.l, v1.h
@@ -219935,78 +220029,79 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_cmp_lg_u32 vcc_lo, 0
; SI-NEXT: v_mul_f32_e64 v42, 1.0, s16
; SI-NEXT: v_mul_f32_e64 v46, 1.0, s41
-; SI-NEXT: s_waitcnt expcnt(6)
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s17
; SI-NEXT: v_mul_f32_e64 v47, 1.0, s40
; SI-NEXT: v_mul_f32_e64 v38, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v45, 1.0, s29
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_mul_f32_e64 v58, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s28
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s19
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v43, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v44, 1.0, s27
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s26
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_mul_f32_e64 v59, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v58, 1.0, s26
; SI-NEXT: v_mul_f32_e64 v55, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v40, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s24
+; SI-NEXT: v_mul_f32_e64 v41, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v53, 1.0, s15
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s14
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, s10
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s37
-; SI-NEXT: v_mul_f32_e64 v35, 1.0, s36
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s35
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s12
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s10
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v62, 1.0, s9
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v35, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s5
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s37
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s36
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s35
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s94
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s31
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s95
; SI-NEXT: v_mul_f32_e64 v22, 1.0, s92
; SI-NEXT: v_mul_f32_e64 v24, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v19, 1.0, s88
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v49, 1.0, s88
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s89
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s77
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s74
-; SI-NEXT: v_mul_f32_e64 v18, 1.0, s75
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s79
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s76
+; SI-NEXT: v_mul_f32_e64 v19, 1.0, s77
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s75
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s58
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s59
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s73
+; SI-NEXT: v_mul_f32_e64 v18, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s63
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s61
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s59
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s56
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s57
; SI-NEXT: v_mul_f32_e64 v52, 1.0, s46
-; SI-NEXT: v_mul_f32_e64 v20, 1.0, s47
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s47
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s44
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s42
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s43
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
@@ -220019,9 +220114,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
@@ -220033,228 +220128,208 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB105_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v42
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v46
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v47
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v28
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v45
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v42
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v31
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v38
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v38
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v43
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v57
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v34
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: v_mov_b32_e32 v31, v38
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v13
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v61
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v52
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v43
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v10
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v33
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v61
+; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v22
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v55
; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v24
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v56
-; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v14
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v44
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v59
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v40
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v19
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v32
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v48
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v36
-; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v53
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v18
-; SI-NEXT: v_mov_b32_e32 v48, v59
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v50
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v62
-; SI-NEXT: v_mov_b32_e32 v43, v34
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v54
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v37
-; SI-NEXT: v_mov_b32_e32 v37, v57
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v35
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v47
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v46
+; SI-NEXT: v_lshrrev_b32_e32 v44, 16, v44
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v8
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v54
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v49
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v58
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v30
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v48
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v45
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v49
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v45, v42
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v36
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v58
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v60
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v19
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v59
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v18
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v39
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v33
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v17
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v10
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v35
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v23
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v56
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v55
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v40
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v41
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v21
+; SI-NEXT: v_mov_b32_e32 v48, v43
+; SI-NEXT: v_mov_b32_e32 v43, v29
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v62
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v34
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v26
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v6, v25
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v57
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v15
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v37
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v50
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v23
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v30
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v52
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v25
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v41
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v57
+; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v32
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v60
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v51
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v33
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v0
-; SI-NEXT: v_mov_b32_e32 v34, v2
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
; SI-NEXT: s_branch .LBB105_3
; SI-NEXT: .LBB105_2:
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v43, v34
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mov_b32_e32 v6, v25
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr42
-; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr43
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr44
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr37
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr56
+; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
@@ -220262,48 +220337,67 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr54
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr39
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr55
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr41
+; SI-NEXT: ; implicit-def: $vgpr61
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr32
-; SI-NEXT: ; implicit-def: $vgpr40
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr1
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
@@ -220311,8 +220405,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
@@ -220320,165 +220414,177 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $vgpr1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr24
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr61
-; SI-NEXT: ; implicit-def: $vgpr17
-; SI-NEXT: ; implicit-def: $vgpr15
-; SI-NEXT: ; implicit-def: $vgpr13
-; SI-NEXT: ; implicit-def: $vgpr11
+; SI-NEXT: ; implicit-def: $vgpr31
+; SI-NEXT: ; implicit-def: $vgpr27
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr29
+; SI-NEXT: ; implicit-def: $vgpr39
; SI-NEXT: ; implicit-def: $vgpr9
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr3
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr56
-; SI-NEXT: ; implicit-def: $vgpr46
-; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: ; implicit-def: $vgpr28
+; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr30
+; SI-NEXT: ; implicit-def: $vgpr42
; SI-NEXT: .LBB105_3: ; %Flow
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v21, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v33, v0
+; SI-NEXT: v_mov_b32_e32 v19, v37
+; SI-NEXT: v_mov_b32_e32 v11, v14
+; SI-NEXT: v_mov_b32_e32 v17, v51
; SI-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; SI-NEXT: v_mov_b32_e32 v14, v46
+; SI-NEXT: v_mov_b32_e32 v40, v44
+; SI-NEXT: v_mov_b32_e32 v59, v55
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_mov_b32_e32 v51, v12
; SI-NEXT: s_cbranch_vccnz .LBB105_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v35, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v34
+; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v45, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
-; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v41
-; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v51
-; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v11
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v39, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v51, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v26
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
+; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: v_add_f32_e32 v37, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v60
+; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v1
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v33
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v1
-; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v26
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v47, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v57
+; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v47, 0x40c00000, v0
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v34
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v0
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v33
-; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v3
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v10
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v3
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v29
+; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v3
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v2
+; SI-NEXT: v_lshr_b64 v[35:36], v[27:28], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[29:30], 16
+; SI-NEXT: v_lshr_b64 v[21:22], v[23:24], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v14, v57
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v42, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v44, 0x40c00000, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v42, 0x40c00000, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v25
+; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v62
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v62
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v41, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v22
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v41
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
@@ -220487,12 +220593,12 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v33
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v19
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
@@ -220501,15 +220607,11 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v1
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v17
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
+; SI-NEXT: v_lshr_b64 v[33:34], v[0:1], 16
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
@@ -220519,12 +220621,13 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v46, 0xffff0000, v15
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220533,12 +220636,12 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220550,9 +220653,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220564,9 +220667,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220592,7 +220695,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
@@ -220606,12 +220709,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v2
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220623,9 +220723,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220637,10 +220737,10 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v4
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220655,7 +220755,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v6
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220670,7 +220770,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v8
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220678,218 +220778,207 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v1
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v10
-; SI-NEXT: v_lshr_b64 v[38:39], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v1, v36
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, v55
+; SI-NEXT: v_and_b32_e32 v53, 0xffff0000, v10
+; SI-NEXT: v_lshr_b64 v[54:55], v[54:55], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v46
-; SI-NEXT: v_lshr_b64 v[35:36], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v36, v60
-; SI-NEXT: v_lshr_b64 v[45:46], v[45:46], 16
-; SI-NEXT: v_mov_b32_e32 v37, v61
-; SI-NEXT: v_and_b32_e32 v34, 0xffff0000, v10
-; SI-NEXT: v_lshr_b64 v[61:62], v[42:43], 16
+; SI-NEXT: v_mov_b32_e32 v1, v28
+; SI-NEXT: v_mov_b32_e32 v28, v60
+; SI-NEXT: v_mov_b32_e32 v29, v61
+; SI-NEXT: v_mov_b32_e32 v61, v52
+; SI-NEXT: v_mov_b32_e32 v60, v51
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v28
-; SI-NEXT: v_lshr_b64 v[27:28], v[27:28], 16
+; SI-NEXT: v_mov_b32_e32 v1, v24
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v59
-; SI-NEXT: v_lshr_b64 v[58:59], v[58:59], 16
-; SI-NEXT: v_mov_b32_e32 v60, v48
-; SI-NEXT: v_mov_b32_e32 v59, v47
-; SI-NEXT: v_mov_b32_e32 v47, v41
-; SI-NEXT: v_mov_b32_e32 v46, v40
-; SI-NEXT: v_mov_b32_e32 v40, v32
-; SI-NEXT: v_mov_b32_e32 v39, v31
-; SI-NEXT: v_mov_b32_e32 v42, v21
-; SI-NEXT: v_mov_b32_e32 v41, v20
-; SI-NEXT: v_mov_b32_e32 v31, v30
-; SI-NEXT: v_mov_b32_e32 v30, v29
-; SI-NEXT: v_mov_b32_e32 v48, v19
-; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v22
-; SI-NEXT: v_lshr_b64 v[21:22], v[30:31], 16
-; SI-NEXT: v_lshr_b64 v[28:29], v[57:58], 16
-; SI-NEXT: v_mov_b32_e32 v56, v27
+; SI-NEXT: v_mov_b32_e32 v1, v30
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v41
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, v40
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[1:2], v[39:40], 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; SI-NEXT: v_lshr_b64 v[3:4], v[52:53], 16
+; SI-NEXT: v_lshr_b64 v[3:4], v[60:61], 16
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
-; SI-NEXT: v_lshr_b64 v[5:6], v[54:55], 16
+; SI-NEXT: v_mov_b32_e32 v5, v26
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[5:6], v[25:26], 16
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; SI-NEXT: v_mov_b32_e32 v7, v50
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[7:8], v[49:50], 16
+; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v13
+; SI-NEXT: v_mov_b32_e32 v13, v48
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; SI-NEXT: v_lshr_b64 v[9:10], v[46:47], 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v11
-; SI-NEXT: v_lshr_b64 v[11:12], v[23:24], 16
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v13
-; SI-NEXT: v_mov_b32_e32 v13, v60
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v9, v38
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[13:14], v[59:60], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v15
-; SI-NEXT: v_lshr_b64 v[15:16], v[36:37], 16
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v17
-; SI-NEXT: v_lshr_b64 v[17:18], v[18:19], 16
-; SI-NEXT: v_mov_b32_e32 v18, v43
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v54, v24
-; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v25
-; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
-; SI-NEXT: v_and_b32_e32 v60, 0xffff0000, v33
+; SI-NEXT: v_lshr_b64 v[9:10], v[37:38], 16
+; SI-NEXT: v_mov_b32_e32 v10, v29
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[50:51], v[47:48], 16
+; SI-NEXT: v_lshr_b64 v[47:48], v[56:57], 16
+; SI-NEXT: v_and_b32_e32 v38, 0xffff0000, v11
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[46:47], 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[23:24], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[10:11], v[49:50], 16
+; SI-NEXT: v_lshr_b64 v[39:40], v[28:29], 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[10:11], v[38:39], 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[27:28], v[44:45], 16
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v19
+; SI-NEXT: v_mov_b32_e32 v48, v59
+; SI-NEXT: v_lshr_b64 v[58:59], v[58:59], 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[20:21], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v14, v45
+; SI-NEXT: v_lshr_b64 v[10:11], v[8:9], 16
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v46, v45
-; SI-NEXT: v_mov_b32_e32 v39, v50
-; SI-NEXT: v_lshr_b64 v[22:23], v[4:5], 16
-; SI-NEXT: v_lshr_b64 v[49:50], v[2:3], 16
-; SI-NEXT: v_lshr_b64 v[32:33], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v59, v58
-; SI-NEXT: v_mov_b32_e32 v36, v35
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[16:17], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[14:15], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[12:13], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[10:11], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[8:9], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[6:7], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[26:27], 16
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[44:45], 16
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[10:11], v[6:7], 16
+; SI-NEXT: v_mov_b32_e32 v45, v32
+; SI-NEXT: v_lshr_b64 v[24:25], v[31:32], 16
+; SI-NEXT: v_lshr_b64 v[31:32], v[42:43], 16
+; SI-NEXT: v_lshr_b64 v[40:41], v[26:27], 16
+; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v12
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[41:42], v[2:3], 16
+; SI-NEXT: v_lshr_b64 v[51:52], v[30:31], 16
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[23:24], 16
+; SI-NEXT: v_lshr_b64 v[59:60], v[57:58], 16
+; SI-NEXT: v_mov_b32_e32 v29, v50
+; SI-NEXT: v_lshr_b64 v[22:23], v[4:5], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[0:1], 16
+; SI-NEXT: v_lshr_b64 v[36:37], v[16:17], 16
+; SI-NEXT: v_lshr_b64 v[10:11], v[20:21], 16
; SI-NEXT: v_lshr_b64 v[18:19], v[34:35], 16
+; SI-NEXT: v_mov_b32_e32 v42, v54
+; SI-NEXT: v_lshr_b64 v[53:54], v[53:54], 16
+; SI-NEXT: v_mov_b32_e32 v56, v33
+; SI-NEXT: v_mov_b32_e32 v28, v17
+; SI-NEXT: v_mov_b32_e32 v38, v21
+; SI-NEXT: v_mov_b32_e32 v30, v35
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: .LBB105_5: ; %end
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v1
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v21
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v14
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v24
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v31
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v5
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v61
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v17
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v27
; SI-NEXT: v_and_b32_e32 v17, 0xffff, v7
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v15
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v59
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v58
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v9
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v13
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v11
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v47
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v29
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v39
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v41
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v53
; SI-NEXT: v_readlane_b32 s37, v63, 5
; SI-NEXT: v_readlane_b32 s36, v63, 4
; SI-NEXT: v_readlane_b32 s35, v63, 3
; SI-NEXT: v_readlane_b32 s34, v63, 2
; SI-NEXT: v_readlane_b32 s31, v63, 1
; SI-NEXT: v_readlane_b32 s30, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v42
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v45
; SI-NEXT: v_or_b32_e32 v1, v2, v1
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v51
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v31
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v43
; SI-NEXT: v_or_b32_e32 v3, v4, v3
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v40
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v56
; SI-NEXT: v_or_b32_e32 v6, v6, v7
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v48
; SI-NEXT: v_or_b32_e32 v7, v8, v7
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v44
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v37
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v25
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v9, v10, v9
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -220899,101 +220988,112 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v11, v12, v11
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v12, v12, v14
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v12, v12, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v54
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v47
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v16, v16, v18
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v16, v16, v18
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v39
; SI-NEXT: v_or_b32_e32 v17, v17, v18
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: v_or_b32_e32 v18, v18, v20
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v18, v18, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v55
; SI-NEXT: v_or_b32_e32 v19, v19, v20
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v24
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v53
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v61
; SI-NEXT: v_or_b32_e32 v21, v21, v22
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v49
; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v40
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v36
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
; SI-NEXT: v_or_b32_e32 v24, v24, v25
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v28
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v29
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v36
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v26, 0xffff, v28
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v27
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v56
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v38
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v28
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v33
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v31
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_or_b32_e32 v28, v28, v29
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v46
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v30
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v33
+; SI-NEXT: v_or_b32_e32 v29, v29, v30
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v42
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -221009,8 +221109,7 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v29, v29, v30
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v38
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v30, v30, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 2123bed663ebb..0509011032c5d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -30379,8 +30379,8 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: s_cbranch_execnz .LBB95_4
; VI-NEXT: .LBB95_2: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
-; VI-NEXT: v_mov_b32_e32 v10, 0x40c00000
-; VI-NEXT: v_add_f32_e32 v0, s4, v10
+; VI-NEXT: v_mov_b32_e32 v12, 0x40c00000
+; VI-NEXT: v_add_f32_e32 v0, s4, v12
; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
@@ -30388,7 +30388,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_add_f32_e32 v1, s4, v10
+; VI-NEXT: v_add_f32_e32 v1, s4, v12
; VI-NEXT: v_bfe_u32 v2, v1, 16, 1
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
@@ -30396,14 +30396,14 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_cndmask_b32_e32 v8, v3, v4, vcc
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
@@ -30413,7 +30413,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; VI-NEXT: s_lshl_b32 s4, s18, 16
; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
@@ -30421,7 +30421,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; VI-NEXT: s_and_b32 s4, s18, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; VI-NEXT: v_add_f32_e32 v3, s4, v10
+; VI-NEXT: v_add_f32_e32 v3, s4, v12
; VI-NEXT: v_bfe_u32 v4, v3, 16, 1
; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; VI-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
@@ -30429,15 +30429,15 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: v_or_b32_e32 v6, 0x400000, v4
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
-; VI-NEXT: v_cndmask_b32_e32 v11, v5, v6, vcc
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
+; VI-NEXT: v_cndmask_b32_e32 v10, v5, v6, vcc
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
@@ -30445,8 +30445,8 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; VI-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
; VI-NEXT: s_lshl_b32 s4, s20, 16
-; VI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
+; VI-NEXT: v_lshrrev_b32_e32 v11, 16, v4
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
@@ -30454,17 +30454,17 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; VI-NEXT: s_and_b32 s4, s20, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; VI-NEXT: v_add_f32_e32 v5, s4, v10
+; VI-NEXT: v_add_f32_e32 v5, s4, v12
; VI-NEXT: v_bfe_u32 v6, v5, 16, 1
; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
; VI-NEXT: s_and_b32 s5, s21, 0xffff0000
; VI-NEXT: v_or_b32_e32 v7, 0x400000, v5
; VI-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; VI-NEXT: v_add_f32_e32 v13, s5, v10
+; VI-NEXT: v_add_f32_e32 v13, s5, v12
; VI-NEXT: s_lshl_b32 s5, s22, 16
; VI-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; VI-NEXT: v_add_f32_e32 v6, s5, v10
+; VI-NEXT: v_add_f32_e32 v6, s5, v12
; VI-NEXT: v_bfe_u32 v7, v6, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
@@ -30472,7 +30472,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; VI-NEXT: s_and_b32 s5, s22, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v6, v7, v14, vcc
-; VI-NEXT: v_add_f32_e32 v7, s5, v10
+; VI-NEXT: v_add_f32_e32 v7, s5, v12
; VI-NEXT: v_bfe_u32 v14, v7, 16, 1
; VI-NEXT: v_add_u32_e32 v14, vcc, v14, v7
; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -30489,7 +30489,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
; VI-NEXT: v_or_b32_e32 v7, 0x400000, v13
; VI-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
-; VI-NEXT: v_add_f32_e32 v13, s4, v10
+; VI-NEXT: v_add_f32_e32 v13, s4, v12
; VI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
; VI-NEXT: v_bfe_u32 v7, v13, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v13
@@ -30499,7 +30499,7 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v13, v7, v14, vcc
; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; VI-NEXT: v_add_f32_e32 v5, s4, v10
+; VI-NEXT: v_add_f32_e32 v5, s4, v12
; VI-NEXT: v_bfe_u32 v7, v5, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v5
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
@@ -30507,25 +30507,25 @@ define inreg <16 x i16> @bitcast_v16bf16_to_v16i16_scalar(<16 x bfloat> inreg %a
; VI-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_cndmask_b32_e32 v5, v7, v15, vcc
-; VI-NEXT: v_add_f32_e32 v7, s4, v10
-; VI-NEXT: v_bfe_u32 v10, v7, 16, 1
-; VI-NEXT: v_add_u32_e32 v10, vcc, v10, v7
-; VI-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
+; VI-NEXT: v_add_f32_e32 v7, s4, v12
+; VI-NEXT: v_bfe_u32 v12, v7, 16, 1
+; VI-NEXT: v_add_u32_e32 v12, vcc, v12, v7
+; VI-NEXT: v_add_u32_e32 v12, vcc, 0x7fff, v12
; VI-NEXT: v_or_b32_e32 v15, 0x400000, v7
; VI-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; VI-NEXT: v_cndmask_b32_e32 v15, v10, v15, vcc
+; VI-NEXT: v_cndmask_b32_e32 v15, v12, v15, vcc
; VI-NEXT: v_lshrrev_b32_e32 v16, 16, v5
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; VI-NEXT: v_lshrrev_b64 v[15:16], 16, v[15:16]
-; VI-NEXT: v_lshrrev_b64 v[13:14], 16, v[13:14]
-; VI-NEXT: v_lshrrev_b64 v[10:11], 16, v[11:12]
+; VI-NEXT: v_lshrrev_b64 v[12:13], 16, v[13:14]
+; VI-NEXT: v_lshrrev_b64 v[10:11], 16, v[10:11]
; VI-NEXT: v_lshrrev_b64 v[7:8], 16, v[8:9]
; VI-NEXT: v_lshrrev_b64 v[2:3], 16, v[2:3]
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v7
; VI-NEXT: v_mov_b32_e32 v3, v10
-; VI-NEXT: v_mov_b32_e32 v5, v13
+; VI-NEXT: v_mov_b32_e32 v5, v12
; VI-NEXT: v_mov_b32_e32 v7, v15
; VI-NEXT: s_setpc_b64 s[30:31]
; VI-NEXT: .LBB95_3:
@@ -32944,7 +32944,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -32954,11 +32954,11 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
@@ -32979,10 +32979,10 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v25, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v20, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v21, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
@@ -33009,11 +33009,11 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
@@ -33022,7 +33022,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
@@ -33056,7 +33056,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
@@ -33090,12 +33090,12 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v25.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v21.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
@@ -35754,8 +35754,8 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: s_cbranch_execnz .LBB103_4
; VI-NEXT: .LBB103_2: ; %cmp.true
; VI-NEXT: s_lshl_b32 s4, s16, 16
-; VI-NEXT: v_mov_b32_e32 v10, 0x40c00000
-; VI-NEXT: v_add_f32_e32 v0, s4, v10
+; VI-NEXT: v_mov_b32_e32 v12, 0x40c00000
+; VI-NEXT: v_add_f32_e32 v0, s4, v12
; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
@@ -35763,7 +35763,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; VI-NEXT: s_and_b32 s4, s16, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_add_f32_e32 v1, s4, v10
+; VI-NEXT: v_add_f32_e32 v1, s4, v12
; VI-NEXT: v_bfe_u32 v2, v1, 16, 1
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
@@ -35771,14 +35771,14 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; VI-NEXT: s_lshl_b32 s4, s17, 16
; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
; VI-NEXT: s_and_b32 s4, s17, 0xffff0000
; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_cndmask_b32_e32 v8, v3, v4, vcc
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
@@ -35788,7 +35788,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; VI-NEXT: s_lshl_b32 s4, s18, 16
; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; VI-NEXT: v_add_f32_e32 v2, s4, v10
+; VI-NEXT: v_add_f32_e32 v2, s4, v12
; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
@@ -35796,7 +35796,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; VI-NEXT: s_and_b32 s4, s18, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
-; VI-NEXT: v_add_f32_e32 v3, s4, v10
+; VI-NEXT: v_add_f32_e32 v3, s4, v12
; VI-NEXT: v_bfe_u32 v4, v3, 16, 1
; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v3
; VI-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
@@ -35804,15 +35804,15 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; VI-NEXT: s_lshl_b32 s4, s19, 16
; VI-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
; VI-NEXT: s_and_b32 s4, s19, 0xffff0000
; VI-NEXT: v_or_b32_e32 v6, 0x400000, v4
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
-; VI-NEXT: v_cndmask_b32_e32 v11, v5, v6, vcc
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
+; VI-NEXT: v_cndmask_b32_e32 v10, v5, v6, vcc
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
@@ -35820,8 +35820,8 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; VI-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
; VI-NEXT: s_lshl_b32 s4, s20, 16
-; VI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; VI-NEXT: v_add_f32_e32 v4, s4, v10
+; VI-NEXT: v_lshrrev_b32_e32 v11, 16, v4
+; VI-NEXT: v_add_f32_e32 v4, s4, v12
; VI-NEXT: v_bfe_u32 v5, v4, 16, 1
; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
@@ -35829,17 +35829,17 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
; VI-NEXT: s_and_b32 s4, s20, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc
-; VI-NEXT: v_add_f32_e32 v5, s4, v10
+; VI-NEXT: v_add_f32_e32 v5, s4, v12
; VI-NEXT: v_bfe_u32 v6, v5, 16, 1
; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v5
; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
; VI-NEXT: s_and_b32 s5, s21, 0xffff0000
; VI-NEXT: v_or_b32_e32 v7, 0x400000, v5
; VI-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; VI-NEXT: v_add_f32_e32 v13, s5, v10
+; VI-NEXT: v_add_f32_e32 v13, s5, v12
; VI-NEXT: s_lshl_b32 s5, s22, 16
; VI-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; VI-NEXT: v_add_f32_e32 v6, s5, v10
+; VI-NEXT: v_add_f32_e32 v6, s5, v12
; VI-NEXT: v_bfe_u32 v7, v6, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v6
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
@@ -35847,7 +35847,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; VI-NEXT: s_and_b32 s5, s22, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v6, v7, v14, vcc
-; VI-NEXT: v_add_f32_e32 v7, s5, v10
+; VI-NEXT: v_add_f32_e32 v7, s5, v12
; VI-NEXT: v_bfe_u32 v14, v7, 16, 1
; VI-NEXT: v_add_u32_e32 v14, vcc, v14, v7
; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -35864,7 +35864,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
; VI-NEXT: v_or_b32_e32 v7, 0x400000, v13
; VI-NEXT: v_cmp_u_f32_e32 vcc, v13, v13
-; VI-NEXT: v_add_f32_e32 v13, s4, v10
+; VI-NEXT: v_add_f32_e32 v13, s4, v12
; VI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
; VI-NEXT: v_bfe_u32 v7, v13, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v13
@@ -35874,7 +35874,7 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: s_and_b32 s4, s23, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v13, v7, v14, vcc
; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; VI-NEXT: v_add_f32_e32 v5, s4, v10
+; VI-NEXT: v_add_f32_e32 v5, s4, v12
; VI-NEXT: v_bfe_u32 v7, v5, 16, 1
; VI-NEXT: v_add_u32_e32 v7, vcc, v7, v5
; VI-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
@@ -35882,25 +35882,25 @@ define inreg <16 x half> @bitcast_v16bf16_to_v16f16_scalar(<16 x bfloat> inreg %
; VI-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; VI-NEXT: s_lshl_b32 s4, s23, 16
; VI-NEXT: v_cndmask_b32_e32 v5, v7, v15, vcc
-; VI-NEXT: v_add_f32_e32 v7, s4, v10
-; VI-NEXT: v_bfe_u32 v10, v7, 16, 1
-; VI-NEXT: v_add_u32_e32 v10, vcc, v10, v7
-; VI-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
+; VI-NEXT: v_add_f32_e32 v7, s4, v12
+; VI-NEXT: v_bfe_u32 v12, v7, 16, 1
+; VI-NEXT: v_add_u32_e32 v12, vcc, v12, v7
+; VI-NEXT: v_add_u32_e32 v12, vcc, 0x7fff, v12
; VI-NEXT: v_or_b32_e32 v15, 0x400000, v7
; VI-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; VI-NEXT: v_cndmask_b32_e32 v15, v10, v15, vcc
+; VI-NEXT: v_cndmask_b32_e32 v15, v12, v15, vcc
; VI-NEXT: v_lshrrev_b32_e32 v16, 16, v5
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; VI-NEXT: v_lshrrev_b64 v[15:16], 16, v[15:16]
-; VI-NEXT: v_lshrrev_b64 v[13:14], 16, v[13:14]
-; VI-NEXT: v_lshrrev_b64 v[10:11], 16, v[11:12]
+; VI-NEXT: v_lshrrev_b64 v[12:13], 16, v[13:14]
+; VI-NEXT: v_lshrrev_b64 v[10:11], 16, v[10:11]
; VI-NEXT: v_lshrrev_b64 v[7:8], 16, v[8:9]
; VI-NEXT: v_lshrrev_b64 v[2:3], 16, v[2:3]
; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v7
; VI-NEXT: v_mov_b32_e32 v3, v10
-; VI-NEXT: v_mov_b32_e32 v5, v13
+; VI-NEXT: v_mov_b32_e32 v5, v12
; VI-NEXT: v_mov_b32_e32 v7, v15
; VI-NEXT: s_setpc_b64 s[30:31]
; VI-NEXT: .LBB103_3:
@@ -38403,7 +38403,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -38413,11 +38413,11 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
@@ -38438,10 +38438,10 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v25, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v20, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v21, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
@@ -38468,11 +38468,11 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
@@ -38481,7 +38481,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
@@ -38515,7 +38515,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
@@ -38549,12 +38549,12 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v25.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v21.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
@@ -42791,7 +42791,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -42801,11 +42801,11 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
@@ -42826,10 +42826,10 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v25, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v20, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v21, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
@@ -42856,11 +42856,11 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
@@ -42869,7 +42869,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
@@ -42903,7 +42903,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
@@ -42937,12 +42937,12 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v25.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v21.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index 9ae6700ac1825..828f3e6a7efbc 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -17937,44 +17937,44 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
@@ -17985,38 +17985,35 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_4
; GFX11-TRUE16-NEXT: .LBB50_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v26, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v34, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v53, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v24, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v36, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v28, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v39, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v51, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v35, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v52, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
@@ -18024,36 +18021,36 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
@@ -18065,10 +18062,8 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
@@ -18085,15 +18080,14 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
@@ -18102,59 +18096,59 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v53.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v39.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v24.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v21.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
@@ -24279,44 +24273,44 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
@@ -24327,38 +24321,35 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_4
; GFX11-TRUE16-NEXT: .LBB62_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB62_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v26, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v34, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v53, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v24, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v36, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v28, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v39, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v51, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v35, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v52, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
@@ -24366,36 +24357,36 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
@@ -24407,10 +24398,8 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
; GFX11-TRUE16-NEXT: .LBB62_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
@@ -24427,15 +24416,14 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
@@ -24444,59 +24432,59 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v53.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v25.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v52.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v39.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v24.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v21.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index cdd050175c8ba..fb2613e55c8e2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -6891,7 +6891,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v63, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s17
; SI-NEXT: v_mul_f32_e64 v62, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v61, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s27
@@ -6908,7 +6908,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
; SI-NEXT: v_mul_f32_e64 v39, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
@@ -6923,15 +6923,15 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: s_cbranch_scc0 .LBB23_4
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v63
; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 16
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v62
; SI-NEXT: v_lshr_b64 v[2:3], v[39:40], 16
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v61
; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v60
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v60
+; SI-NEXT: v_lshr_b64 v[4:5], v[48:49], 16
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v59
; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v58
@@ -6964,7 +6964,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -6983,7 +6983,7 @@ define inreg <16 x i32> @bitcast_v32bf16_to_v16i32_scalar(<32 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -21659,7 +21659,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v63, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s17
; SI-NEXT: v_mul_f32_e64 v62, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v61, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s27
@@ -21676,7 +21676,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
; SI-NEXT: v_mul_f32_e64 v39, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
@@ -21691,15 +21691,15 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: s_cbranch_scc0 .LBB47_4
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v63
; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 16
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v62
; SI-NEXT: v_lshr_b64 v[2:3], v[39:40], 16
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v61
; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v60
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v60
+; SI-NEXT: v_lshr_b64 v[4:5], v[48:49], 16
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v59
; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v58
@@ -21732,7 +21732,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -21751,7 +21751,7 @@ define inreg <16 x float> @bitcast_v32bf16_to_v16f32_scalar(<32 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -35785,7 +35785,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v63, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s17
; SI-NEXT: v_mul_f32_e64 v62, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v61, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s27
@@ -35802,7 +35802,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
; SI-NEXT: v_mul_f32_e64 v39, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
@@ -35817,15 +35817,15 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: s_cbranch_scc0 .LBB67_4
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v63
; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 16
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v62
; SI-NEXT: v_lshr_b64 v[2:3], v[39:40], 16
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v61
; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v60
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v60
+; SI-NEXT: v_lshr_b64 v[4:5], v[48:49], 16
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v59
; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v58
@@ -35858,7 +35858,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -35877,7 +35877,7 @@ define inreg <8 x i64> @bitcast_v32bf16_to_v8i64_scalar(<32 x bfloat> inreg %a,
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -48991,7 +48991,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v63, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s17
; SI-NEXT: v_mul_f32_e64 v62, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v61, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v60, 1.0, s27
@@ -49008,7 +49008,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v16, 1.0, s43
; SI-NEXT: v_mul_f32_e64 v39, 1.0, s18
; SI-NEXT: v_mul_f32_e64 v54, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s20
; SI-NEXT: v_mul_f32_e64 v37, 1.0, s21
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s23
@@ -49023,15 +49023,15 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: s_cbranch_scc0 .LBB83_4
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v41
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v63
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v63
; SI-NEXT: v_lshr_b64 v[0:1], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[1:2], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[50:51], 16
; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v62
; SI-NEXT: v_lshr_b64 v[2:3], v[39:40], 16
; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v61
; SI-NEXT: v_lshr_b64 v[3:4], v[54:55], 16
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v60
-; SI-NEXT: v_lshr_b64 v[4:5], v[50:51], 16
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v60
+; SI-NEXT: v_lshr_b64 v[4:5], v[48:49], 16
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v59
; SI-NEXT: v_lshr_b64 v[5:6], v[37:38], 16
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v58
@@ -49064,7 +49064,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v63
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -49083,7 +49083,7 @@ define inreg <8 x double> @bitcast_v32bf16_to_v8f64_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v50
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v48
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
@@ -68872,16 +68872,16 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
@@ -68899,44 +68899,42 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v20.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
@@ -68944,13 +68942,13 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
@@ -68964,35 +68962,35 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v49, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v71, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v66, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v86, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v28, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v24, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v70, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v80, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v85, v81, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
@@ -69000,63 +68998,63 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v83, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v20.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v38, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
@@ -69065,43 +69063,43 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
@@ -69113,14 +69111,14 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
@@ -69128,7 +69126,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
@@ -69137,7 +69135,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
@@ -69146,7 +69144,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
@@ -69156,7 +69154,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
@@ -69170,16 +69168,16 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v69.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
@@ -69187,51 +69185,51 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
@@ -81630,16 +81628,16 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
@@ -81657,44 +81655,42 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v20.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
@@ -81702,13 +81698,13 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
@@ -81722,35 +81718,35 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v49, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v71, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v66, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v86, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v28, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v24, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v70, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v80, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v85, v81, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
@@ -81758,63 +81754,63 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v83, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v20.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v38, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
@@ -81823,43 +81819,43 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
@@ -81871,14 +81867,14 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
@@ -81886,7 +81882,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
@@ -81895,7 +81891,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
@@ -81904,7 +81900,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
@@ -81914,7 +81910,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
@@ -81928,16 +81924,16 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v69.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
@@ -81945,51 +81941,51 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
@@ -92418,16 +92414,16 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:116
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
@@ -92445,44 +92441,42 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v30.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v23.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v20.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
@@ -92490,13 +92484,13 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
@@ -92510,35 +92504,35 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v50, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v69, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v64, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v85, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v52, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v49, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v71, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v66, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v86, v68, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v82, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v28, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v87, v69, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v54, v24, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v36.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v67, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v70, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v83, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v96, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v33.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v80, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v86, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v85, v81, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v10.l
; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v9.l, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h
@@ -92546,63 +92540,63 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v19.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v20.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.l
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.h, v19.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v32.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v34.l
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v16.l
; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.l, v21.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v97, v83, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v18.h
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v17.h
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v99, v98, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
-; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v49, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.h, v20.l
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v18.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v38, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v15.l, v21.l
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.h, v20.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v66, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v53, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v16.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
@@ -92611,43 +92605,43 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v53.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v38.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v82.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, 0x300, v0.h
@@ -92659,14 +92653,14 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v20.l, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v99.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v20.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v36.l, 3
@@ -92674,7 +92668,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v98.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v36.h, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.l, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
@@ -92683,7 +92677,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v18.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v97.l, 3
@@ -92692,7 +92686,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v84.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v83.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.h, 3
@@ -92702,7 +92696,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v17.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v86.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v85.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v16.l, v1.l
@@ -92716,16 +92710,16 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v69.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v83.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v85.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v84.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v86.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v71.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v80.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v65.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v0.l
@@ -92733,51 +92727,51 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v67.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v64.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v64.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v66.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v38.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v28.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v52.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v54.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v23.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v22.l
; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.l, v1.h
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 0ee94a1bb7d05..1414f01593783 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -2204,31 +2204,31 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_mov_b32 s11, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB4_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
+; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[6:7]
+; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: s_mul_u64 s[6:7], s[4:5], s[10:11]
+; GFX1264-NEXT: s_mov_b32 s14, -1
; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v0, s8
-; GFX1264-NEXT: v_mov_b32_e32 v1, s9
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_mov_b32 s8, s2
-; GFX1264-NEXT: s_mov_b32 s9, s3
-; GFX1264-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s12, s2
+; GFX1264-NEXT: s_mov_b32 s13, s3
+; GFX1264-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB4_2:
-; GFX1264-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: v_readfirstlane_b32 s3, v1
; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
@@ -6029,15 +6029,17 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
+; GFX7LESS-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mul_i32 s14, s9, s6
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
@@ -6046,11 +6048,9 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s14, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s13
; GFX7LESS-NEXT: s_mov_b32 s6, -1
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v10, v1
@@ -6066,14 +6066,13 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
-; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
-; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
@@ -6096,26 +6095,26 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v6, s7, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8-NEXT: s_cbranch_execz .LBB10_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s8, v0, 0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_mul_i32 s6, s9, s6
-; GFX8-NEXT: s_mov_b64 s[12:13], 0
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v5
+; GFX8-NEXT: s_bcnt1_i32_b64 s10, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[6:7], s8, v0, 0
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX8-NEXT: s_mul_i32 s10, s9, s10
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s10, v5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: s_mov_b32 s4, s2
-; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v10, v1
@@ -6130,14 +6129,13 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
-; GFX8-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB10_2
; GFX8-NEXT: ; %bb.3: ; %Flow
-; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
-; GFX8-NEXT: .LBB10_4: ; %Flow4
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: .LBB10_4: ; %Flow4
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_mul_lo_u32 v4, s9, v6
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v6, 0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
@@ -6158,32 +6156,32 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_4
; GFX9-NEXT: ; %bb.1:
+; GFX9-NEXT: s_bcnt1_i32_b64 s10, s[6:7]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX9-NEXT: s_mul_i32 s11, s9, s10
+; GFX9-NEXT: s_mul_hi_u32 s12, s8, s10
+; GFX9-NEXT: s_add_i32 s13, s12, s11
+; GFX9-NEXT: s_mul_i32 s12, s8, s10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX9-NEXT: s_mul_i32 s7, s9, s6
-; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
-; GFX9-NEXT: s_add_i32 s7, s12, s7
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: s_mul_i32 s14, s8, s6
-; GFX9-NEXT: s_mov_b64 s[12:13], 0
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s13
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
-; GFX9-NEXT: s_mov_b32 s4, s2
-; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v9, v1
; GFX9-NEXT: v_mov_b32_e32 v8, v0
-; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s14, v8
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s12, v8
; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
; GFX9-NEXT: v_mov_b32_e32 v0, v6
; GFX9-NEXT: v_mov_b32_e32 v1, v7
@@ -6193,14 +6191,13 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX9-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB10_2
; GFX9-NEXT: ; %bb.3: ; %Flow
-; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
-; GFX9-NEXT: .LBB10_4: ; %Flow4
; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: .LBB10_4: ; %Flow4
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v4, 0
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
@@ -6223,30 +6220,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1064-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b64 s[12:13], 0
; GFX1064-NEXT: s_mul_i32 s7, s9, s6
-; GFX1064-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1064-NEXT: s_mul_i32 s14, s8, s6
-; GFX1064-NEXT: s_add_i32 s15, s15, s7
+; GFX1064-NEXT: s_mul_hi_u32 s12, s8, s6
+; GFX1064-NEXT: s_mul_i32 s13, s8, s6
+; GFX1064-NEXT: s_add_i32 s12, s12, s7
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
-; GFX1064-NEXT: v_mov_b32_e32 v1, s5
-; GFX1064-NEXT: s_mov_b32 s4, s2
-; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: v_mov_b32_e32 v0, s10
+; GFX1064-NEXT: v_mov_b32_e32 v1, s11
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v8, v1
; GFX1064-NEXT: v_mov_b32_e32 v7, v0
-; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s14
-; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s15, v8, vcc
+; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s13
+; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s12, v8, vcc
; GFX1064-NEXT: v_mov_b32_e32 v0, v5
; GFX1064-NEXT: v_mov_b32_e32 v2, v7
; GFX1064-NEXT: v_mov_b32_e32 v1, v6
@@ -6256,14 +6253,13 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
-; GFX1064-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_cbranch_execnz .LBB10_2
; GFX1064-NEXT: ; %bb.3: ; %Flow
-; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
-; GFX1064-NEXT: .LBB10_4: ; %Flow4
; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: .LBB10_4: ; %Flow4
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s8, v4, 0
; GFX1064-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s9, v4, v[3:4]
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
@@ -6281,33 +6277,33 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; GFX1032-NEXT: s_mov_b32 s6, exec_lo
-; GFX1032-NEXT: s_mov_b32 s11, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX1032-NEXT: s_and_saveexec_b32 s10, vcc_lo
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1032-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_mul_i32 s7, s9, s6
-; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1032-NEXT: s_mul_i32 s12, s8, s6
-; GFX1032-NEXT: s_add_i32 s13, s13, s7
+; GFX1032-NEXT: s_mul_hi_u32 s10, s8, s6
+; GFX1032-NEXT: s_mul_i32 s11, s8, s6
+; GFX1032-NEXT: s_add_i32 s10, s10, s7
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
-; GFX1032-NEXT: s_mov_b32 s4, s2
-; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: v_mov_b32_e32 v0, s12
+; GFX1032-NEXT: v_mov_b32_e32 v1, s13
; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v8, v1
; GFX1032-NEXT: v_mov_b32_e32 v7, v0
-; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s12
-; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s13, v8, vcc_lo
+; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s11
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s10, v8, vcc_lo
; GFX1032-NEXT: v_mov_b32_e32 v0, v5
; GFX1032-NEXT: v_mov_b32_e32 v2, v7
; GFX1032-NEXT: v_mov_b32_e32 v1, v6
@@ -6317,14 +6313,13 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
-; GFX1032-NEXT: s_or_b32 s11, vcc_lo, s11
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB10_2
; GFX1032-NEXT: ; %bb.3: ; %Flow
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: .LBB10_4: ; %Flow4
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s2, s8, v4, 0
; GFX1032-NEXT: v_readfirstlane_b32 s3, v1
; GFX1032-NEXT: v_mad_u64_u32 v[3:4], s2, s9, v4, v[3:4]
@@ -6342,41 +6337,42 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: s_mov_b64 s[10:11], exec
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-NEXT: s_cbranch_execz .LBB10_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1164-NEXT: s_load_b64 s[10:11], s[4:5], 0x0
; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: s_mul_i32 s7, s9, s6
-; GFX1164-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1164-NEXT: s_mul_i32 s14, s8, s6
-; GFX1164-NEXT: s_add_i32 s15, s15, s7
+; GFX1164-NEXT: s_mul_hi_u32 s12, s8, s6
+; GFX1164-NEXT: s_mul_i32 s13, s8, s6
+; GFX1164-NEXT: s_add_i32 s12, s12, s7
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
-; GFX1164-NEXT: v_mov_b32_e32 v1, s5
-; GFX1164-NEXT: s_mov_b32 s4, s2
-; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: v_mov_b32_e32 v0, s10
+; GFX1164-NEXT: v_mov_b32_e32 v1, s11
+; GFX1164-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164-NEXT: v_mov_b32_e32 v8, v1
; GFX1164-NEXT: v_mov_b32_e32 v7, v0
-; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s14
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s15, v8, vcc
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s13
+; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s12, v8, vcc
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1164-NEXT: v_mov_b32_e32 v0, v5
; GFX1164-NEXT: v_mov_b32_e32 v2, v7
; GFX1164-NEXT: v_mov_b32_e32 v3, v8
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_mov_b32_e32 v1, v6
; GFX1164-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
@@ -6384,16 +6380,15 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX1164-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[12:13]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-NEXT: s_cbranch_execnz .LBB10_2
; GFX1164-NEXT: ; %bb.3: ; %Flow
-; GFX1164-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1164-NEXT: .LBB10_4: ; %Flow4
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 0
; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-NEXT: v_readfirstlane_b32 s3, v1
@@ -6412,35 +6407,35 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: s_mov_b32 s11, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
-; GFX1132-NEXT: s_mov_b32 s10, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB10_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1132-NEXT: s_load_b64 s[12:13], s[4:5], 0x0
; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s7, s9, s6
-; GFX1132-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1132-NEXT: s_mul_i32 s12, s8, s6
-; GFX1132-NEXT: s_add_i32 s13, s13, s7
+; GFX1132-NEXT: s_mul_hi_u32 s10, s8, s6
+; GFX1132-NEXT: s_mul_i32 s11, s8, s6
+; GFX1132-NEXT: s_add_i32 s10, s10, s7
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX1132-NEXT: s_mov_b32 s4, s2
-; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s12
+; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s11
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s13, v8, vcc_lo
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s10, v8, vcc_lo
; GFX1132-NEXT: v_mov_b32_e32 v0, v5
; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
@@ -6450,16 +6445,15 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
-; GFX1132-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s11
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB10_2
; GFX1132-NEXT: ; %bb.3: ; %Flow
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: .LBB10_4: ; %Flow4
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 0
; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132-NEXT: v_readfirstlane_b32 s3, v1
@@ -6477,31 +6471,31 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_mov_b32 s11, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB10_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
+; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[6:7]
+; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: s_mul_u64 s[6:7], s[4:5], s[10:11]
+; GFX1264-NEXT: s_mov_b32 s14, -1
; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v0, s8
-; GFX1264-NEXT: v_mov_b32_e32 v1, s9
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_mov_b32 s8, s2
-; GFX1264-NEXT: s_mov_b32 s9, s3
-; GFX1264-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s12, s2
+; GFX1264-NEXT: s_mov_b32 s13, s3
+; GFX1264-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB10_2:
-; GFX1264-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
; GFX1264-NEXT: v_readfirstlane_b32 s2, v0
@@ -8104,7 +8098,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-LABEL: uniform_or_i8:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s14, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -8116,38 +8110,39 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xff
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
-; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_and_b32 s1, s14, 0xff
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
-; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s13
+; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_lshl_b32 s15, s1, s10
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX7LESS-NEXT: .LBB12_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v0
; GFX7LESS-NEXT: buffer_store_byte v0, off, s[8:11], 0
@@ -8156,7 +8151,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-LABEL: uniform_or_i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -8169,34 +8164,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
-; GFX8-NEXT: s_and_b32 s0, s12, 0xff
-; GFX8-NEXT: s_lshl_b32 s14, s0, s13
-; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_lshl_b32 s10, s0, 3
+; GFX8-NEXT: s_and_b32 s0, s14, 0xff
+; GFX8-NEXT: s_lshl_b32 s15, s0, s10
+; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB12_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: .LBB12_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
@@ -8207,7 +8202,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-LABEL: uniform_or_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -8220,34 +8215,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
-; GFX9-NEXT: s_and_b32 s0, s12, 0xff
-; GFX9-NEXT: s_lshl_b32 s14, s0, s13
-; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_lshl_b32 s10, s0, 3
+; GFX9-NEXT: s_and_b32 s0, s14, 0xff
+; GFX9-NEXT: s_lshl_b32 s15, s0, s10
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: .LBB12_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v0, s14
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
@@ -8259,7 +8254,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -8272,34 +8267,34 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1064-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1064-NEXT: .LBB12_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
@@ -8321,18 +8316,19 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_and_b32 s6, s10, 3
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, s1, 0xff
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s5, s10, 3
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1032-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -8360,7 +8356,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8374,10 +8370,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -8385,26 +8381,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-TRUE16-NEXT: .LBB12_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
@@ -8415,7 +8411,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8429,10 +8425,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -8440,26 +8436,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-FAKE16-NEXT: .LBB12_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
@@ -8482,19 +8478,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -8535,19 +8533,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -8576,7 +8576,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8590,10 +8590,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -8601,26 +8601,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-TRUE16-NEXT: .LBB12_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -8633,7 +8633,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8647,10 +8647,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -8658,26 +8658,26 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-FAKE16-NEXT: .LBB12_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -8702,19 +8702,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -8757,19 +8759,21 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -8805,7 +8809,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
@@ -8817,45 +8821,46 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_and_b32 s11, s2, 3
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s6
-; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 3
-; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x0
-; GFX7LESS-NEXT: s_lshl_b32 s12, 0xff, s11
+; GFX7LESS-NEXT: s_mul_i32 s6, s12, s6
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s13, 0xff, s2
; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff
-; GFX7LESS-NEXT: s_not_b32 s13, s12
-; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s11
+; GFX7LESS-NEXT: s_not_b32 s14, s13
+; GFX7LESS-NEXT: s_lshl_b32 s15, s6, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s15, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s13, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s14, v1
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX7LESS-NEXT: .LBB13_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
-; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xff
+; GFX7LESS-NEXT: s_and_b32 s5, s12, 0xff
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v0
; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -8864,7 +8869,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-LABEL: uniform_add_i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -8875,39 +8880,40 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_mul_i32 s6, s12, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s11, s2, 3
-; GFX8-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX8-NEXT: s_and_b32 s2, s6, 0xff
-; GFX8-NEXT: s_not_b32 s13, s12
-; GFX8-NEXT: s_lshl_b32 s14, s2, s11
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_not_b32 s14, s13
+; GFX8-NEXT: s_lshl_b32 s15, s6, s2
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
-; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s15, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s13, v0
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB13_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX8-NEXT: .LBB13_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_readfirstlane_b32 s4, v0
@@ -8915,14 +8921,14 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: v_mad_u16 v0, s12, v4, v0
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_add_i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -8933,38 +8939,39 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_mul_i32 s6, s12, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s11, s2, 3
-; GFX9-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX9-NEXT: s_and_b32 s2, s6, 0xff
-; GFX9-NEXT: s_not_b32 s13, s12
-; GFX9-NEXT: s_lshl_b32 s14, s2, s11
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_not_b32 s14, s13
+; GFX9-NEXT: s_lshl_b32 s15, s6, s2
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
-; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_add_u32_e32 v0, s15, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX9-NEXT: .LBB13_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -8972,7 +8979,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, s12, v4, v0
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -8980,7 +8987,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -8993,42 +9000,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
-; GFX1064-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1064-NEXT: s_not_b32 s13, s12
-; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1064-NEXT: s_not_b32 s14, s13
+; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: .LBB13_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX1064-NEXT: s_endpgm
@@ -9050,23 +9058,24 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_mul_i32 s6, s8, s6
-; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1032-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1032-NEXT: s_not_b32 s11, s3
-; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_not_b32 s12, s11
+; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s11, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -9093,7 +9102,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -9107,26 +9116,27 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -9134,20 +9144,20 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: .LBB13_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
; GFX1164-TRUE16-NEXT: s_endpgm
@@ -9156,7 +9166,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -9170,26 +9180,27 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
@@ -9197,20 +9208,20 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: .LBB13_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
; GFX1164-FAKE16-NEXT: s_endpgm
@@ -9233,25 +9244,26 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -9293,25 +9305,26 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -9339,7 +9352,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -9353,40 +9366,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: .LBB13_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -9394,7 +9410,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1264-TRUE16-NEXT: s_endpgm
@@ -9403,7 +9419,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -9417,40 +9433,43 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
-; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB13_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: .LBB13_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -9458,7 +9477,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1264-FAKE16-NEXT: s_endpgm
@@ -9481,28 +9500,29 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
+; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -9545,28 +9565,29 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
+; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -9606,18 +9627,19 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xff
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s11, 0xff, s10
-; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s10
-; GFX7LESS-NEXT: s_not_b32 s3, s11
+; GFX7LESS-NEXT: s_lshl_b32 s2, s6, s10
+; GFX7LESS-NEXT: s_not_b32 s11, s11
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s11, v1
; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
@@ -9641,39 +9663,40 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s8, s2, 3
-; GFX8-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX8-NEXT: s_not_b32 s9, s2
-; GFX8-NEXT: s_and_b32 s2, s6, 0xff
-; GFX8-NEXT: s_lshl_b32 s10, s2, s8
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s10, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xff, s10
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: s_lshl_b32 s11, s6, s10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -9681,39 +9704,40 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s8, s2, 3
-; GFX9-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX9-NEXT: s_not_b32 s9, s2
-; GFX9-NEXT: s_and_b32 s2, s6, 0xff
-; GFX9-NEXT: s_lshl_b32 s10, s2, s8
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s10, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xff, s10
+; GFX9-NEXT: s_and_b32 s6, s6, 0xff
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_lshl_b32 s11, s6, s10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB14_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -9722,36 +9746,37 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1064-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1064-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1064-NEXT: s_not_b32 s10, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -9767,19 +9792,20 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1032-NEXT: s_not_b32 s8, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -9802,38 +9828,39 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_and_b32 s4, s2, -4
; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1164-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1164-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1164-NEXT: s_not_b32 s10, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_1
; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 0
@@ -9849,20 +9876,21 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1132-NEXT: s_and_b32 s4, s2, -4
; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1132-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1132-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1132-NEXT: s_not_b32 s8, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -9885,38 +9913,39 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_and_b32 s4, s2, -4
; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1264-NEXT: s_lshl_b32 s2, 0xff, s8
-; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1264-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1264-NEXT: s_not_b32 s10, s7
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-NEXT: s_mov_b32 s6, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1264-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_cbranch_execnz .LBB14_1
; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null
@@ -9932,20 +9961,21 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1232-NEXT: s_and_b32 s4, s2, -4
; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-NEXT: s_and_b32 s6, s6, 0xff
-; GFX1232-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_lshl_b32 s7, 0xff, s2
+; GFX1232-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1232-NEXT: s_not_b32 s8, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-NEXT: s_mov_b32 s6, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -9971,7 +10001,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-LABEL: uniform_or_i16:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s14, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -9983,38 +10013,39 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xffff
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
-; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_and_b32 s1, s14, 0xffff
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
-; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s13
+; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_lshl_b32 s15, s1, s10
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s13, 16
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s10, 16
; GFX7LESS-NEXT: .LBB15_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v0
; GFX7LESS-NEXT: buffer_store_short v0, off, s[8:11], 0
@@ -10023,7 +10054,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-LABEL: uniform_or_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -10036,34 +10067,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s12
-; GFX8-NEXT: s_lshl_b32 s14, s0, s13
-; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_lshl_b32 s10, s0, 3
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX8-NEXT: s_lshl_b32 s15, s0, s10
+; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB15_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: .LBB15_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
@@ -10074,7 +10105,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-LABEL: uniform_or_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -10087,34 +10118,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
-; GFX9-NEXT: s_and_b32 s0, 0xffff, s12
-; GFX9-NEXT: s_lshl_b32 s14, s0, s13
-; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_lshl_b32 s10, s0, 3
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX9-NEXT: s_lshl_b32 s15, s0, s10
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
-; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: .LBB15_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v0, s14
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
@@ -10126,7 +10157,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s14, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -10139,34 +10170,34 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1064-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1064-NEXT: .LBB15_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
@@ -10188,18 +10219,19 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_and_b32 s6, s10, 3
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s5, s10, 3
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1032-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -10227,7 +10259,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10241,10 +10273,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -10252,26 +10284,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-TRUE16-NEXT: .LBB15_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
@@ -10282,7 +10314,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10296,10 +10328,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
@@ -10307,26 +10339,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1164-FAKE16-NEXT: .LBB15_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
@@ -10349,19 +10381,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -10402,19 +10436,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -10443,7 +10479,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10457,10 +10493,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -10468,26 +10504,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-TRUE16-NEXT: .LBB15_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s14, 0, vcc
; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -10500,7 +10536,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s14, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10514,10 +10550,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -10525,26 +10561,26 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s15, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[12:13], s[0:1], s[12:13]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[12:13]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX1264-FAKE16-NEXT: .LBB15_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s14, 0, vcc
; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -10569,19 +10605,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -10624,19 +10662,21 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s12, v1
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -10672,7 +10712,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
@@ -10684,45 +10724,46 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_and_b32 s11, s2, 3
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s6
-; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 3
-; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x0
-; GFX7LESS-NEXT: s_lshl_b32 s12, 0xffff, s11
+; GFX7LESS-NEXT: s_mul_i32 s6, s12, s6
+; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX7LESS-NEXT: s_lshl_b32 s13, 0xffff, s2
; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX7LESS-NEXT: s_not_b32 s13, s12
-; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s11
+; GFX7LESS-NEXT: s_not_b32 s14, s13
+; GFX7LESS-NEXT: s_lshl_b32 s15, s6, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s15, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s13, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s14, v1
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s11, 16
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s2, 16
; GFX7LESS-NEXT: .LBB16_4: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
-; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xffff
+; GFX7LESS-NEXT: s_and_b32 s5, s12, 0xffff
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v0
; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -10731,7 +10772,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-LABEL: uniform_add_i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10742,39 +10783,40 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_mul_i32 s6, s12, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s11, s2, 3
-; GFX8-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX8-NEXT: s_not_b32 s13, s12
-; GFX8-NEXT: s_lshl_b32 s14, s2, s11
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 3
+; GFX8-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX8-NEXT: s_not_b32 s14, s13
+; GFX8-NEXT: s_lshl_b32 s15, s6, s2
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v1
-; GFX8-NEXT: v_and_b32_e32 v2, s13, v1
-; GFX8-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s15, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s14, v1
+; GFX8-NEXT: v_and_b32_e32 v0, s13, v0
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB16_2
; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX8-NEXT: .LBB16_4: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_readfirstlane_b32 s4, v0
@@ -10782,14 +10824,14 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mad_u16 v0, s10, v4, v0
+; GFX8-NEXT: v_mad_u16 v0, s12, v4, v0
; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_add_i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10800,38 +10842,39 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_mul_i32 s6, s12, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s11, s2, 3
-; GFX9-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX9-NEXT: s_not_b32 s13, s12
-; GFX9-NEXT: s_lshl_b32 s14, s2, s11
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 3
+; GFX9-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX9-NEXT: s_not_b32 s14, s13
+; GFX9-NEXT: s_lshl_b32 s15, s6, s2
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_u32_e32 v0, s14, v1
-; GFX9-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX9-NEXT: v_add_u32_e32 v0, s15, v1
+; GFX9-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX9-NEXT: .LBB16_4: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -10839,7 +10882,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v0
+; GFX9-NEXT: v_mad_legacy_u16 v0, s12, v4, v0
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -10847,7 +10890,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
@@ -10860,42 +10903,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
-; GFX1064-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1064-NEXT: s_not_b32 s13, s12
-; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1064-NEXT: s_not_b32 s14, s13
+; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1064-NEXT: v_and_b32_e32 v0, s12, v0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1064-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1064-NEXT: v_and_b32_e32 v0, s13, v0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: .LBB16_4: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1064-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX1064-NEXT: s_endpgm
@@ -10917,23 +10961,24 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_mul_i32 s6, s8, s6
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1032-NEXT: s_not_b32 s11, s3
-; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1032-NEXT: s_not_b32 s12, s11
+; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1032-NEXT: v_and_b32_e32 v0, s3, v0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1032-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1032-NEXT: v_and_b32_e32 v0, s11, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -10960,7 +11005,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -10974,26 +11019,27 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -11001,20 +11047,20 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: .LBB16_4: ; %Flow
; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX1164-TRUE16-NEXT: s_endpgm
@@ -11023,7 +11069,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -11037,26 +11083,27 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
@@ -11064,20 +11111,20 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: .LBB16_4: ; %Flow
; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX1164-FAKE16-NEXT: s_endpgm
@@ -11100,25 +11147,26 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -11160,25 +11208,26 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -11206,7 +11255,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -11220,40 +11269,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
+; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: .LBB16_4: ; %Flow
; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -11261,7 +11313,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s2
+; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s12, v4.l, s2
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1264-TRUE16-NEXT: s_endpgm
@@ -11270,7 +11322,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
@@ -11284,40 +11336,43 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
+; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v1
-; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v0
+; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s15, v1
+; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s13, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s14, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB16_2
; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: .LBB16_4: ; %Flow
; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -11325,7 +11380,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s2
+; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s12, v4, s2
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1264-FAKE16-NEXT: s_endpgm
@@ -11348,28 +11403,29 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
+; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -11412,28 +11468,29 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
+; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v0
+; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s13, v1
+; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s11, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s12, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -11473,18 +11530,19 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xffff
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s11, 0xffff, s10
-; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s10
-; GFX7LESS-NEXT: s_not_b32 s3, s11
+; GFX7LESS-NEXT: s_lshl_b32 s2, s6, s10
+; GFX7LESS-NEXT: s_not_b32 s11, s11
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v1
+; GFX7LESS-NEXT: v_and_b32_e32 v0, s11, v1
; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
@@ -11508,39 +11566,40 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_lshl_b32 s8, s2, 3
-; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX8-NEXT: s_not_b32 s9, s2
-; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s10, s2, s8
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_lshl_b32 s10, s2, 3
+; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX8-NEXT: s_not_b32 s2, s2
+; GFX8-NEXT: s_lshl_b32 s11, s6, s10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX8-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX8-NEXT: v_or_b32_e32 v0, s11, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -11548,39 +11607,40 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[8:9], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_lshl_b32 s8, s2, 3
-; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX9-NEXT: s_not_b32 s9, s2
-; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
-; GFX9-NEXT: s_lshl_b32 s10, s2, s8
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_lshl_b32 s10, s2, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX9-NEXT: s_not_b32 s2, s2
+; GFX9-NEXT: s_lshl_b32 s11, s6, s10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_and_b32_e32 v0, s9, v1
-; GFX9-NEXT: v_or_b32_e32 v0, s10, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s2, v1
+; GFX9-NEXT: v_or_b32_e32 v0, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB17_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v2
; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -11589,36 +11649,37 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1064-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1064-NEXT: s_not_b32 s9, s2
+; GFX1064-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1064-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1064-NEXT: s_not_b32 s10, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -11634,19 +11695,20 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1032-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
+; GFX1032-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1032-NEXT: s_not_b32 s8, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -11669,38 +11731,39 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_and_b32 s4, s2, -4
; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1164-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1164-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1164-NEXT: s_not_b32 s9, s2
+; GFX1164-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1164-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1164-NEXT: s_not_b32 s10, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1164-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_1
; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -11716,20 +11779,21 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-NEXT: s_and_b32 s4, s2, -4
; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1132-NEXT: s_not_b32 s3, s3
+; GFX1132-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1132-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1132-NEXT: s_not_b32 s8, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v1, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1132-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -11752,38 +11816,39 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_and_b32 s4, s2, -4
; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-NEXT: s_lshl_b32 s8, s2, 3
+; GFX1264-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s8
-; GFX1264-NEXT: s_lshl_b32 s10, s6, s8
-; GFX1264-NEXT: s_not_b32 s9, s2
+; GFX1264-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1264-NEXT: s_lshl_b32 s11, s6, s2
+; GFX1264-NEXT: s_not_b32 s10, s7
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-NEXT: s_mov_b32 s6, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-NEXT: s_mov_b32 s5, s3
; GFX1264-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s10
+; GFX1264-NEXT: v_and_or_b32 v0, v1, s10, s11
; GFX1264-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_cbranch_execnz .LBB17_1
; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -11799,20 +11864,21 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-NEXT: s_and_b32 s4, s2, -4
; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-NEXT: s_lshl_b32 s8, s6, s2
-; GFX1232-NEXT: s_not_b32 s3, s3
+; GFX1232-NEXT: s_lshl_b32 s7, 0xffff, s2
+; GFX1232-NEXT: s_lshl_b32 s10, s6, s2
+; GFX1232-NEXT: s_not_b32 s8, s7
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-NEXT: s_mov_b32 s6, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_mov_b32_e32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s8
+; GFX1232-NEXT: v_and_or_b32 v0, v1, s8, s10
; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -11845,14 +11911,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, s6
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -11890,13 +11957,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
; GFX8-NEXT: s_lshl_b32 s10, s2, 3
; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX8-NEXT: s_not_b32 s2, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -11931,13 +11999,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
; GFX9-NEXT: s_lshl_b32 s10, s2, 3
; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX9-NEXT: s_not_b32 s2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -11965,38 +12034,39 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s9, s2, 3
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1064-NEXT: s_not_b32 s10, s2
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1064-NEXT: s_not_b32 s11, s6
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1064-NEXT: v_add_f16_e32 v0, s8, v0
-; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1064-NEXT: v_add_f16_e32 v0, s10, v0
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB18_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -12013,19 +12083,20 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1032-NEXT: s_not_b32 s3, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s6
+; GFX1032-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1032-NEXT: s_not_b32 s10, s6
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1032-NEXT: v_add_f16_e32 v0, s8, v0
; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -12047,30 +12118,31 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1164-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1164-TRUE16-NEXT: s_not_b32 s11, s6
+; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
+; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s10, v0.l
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -12078,13 +12150,13 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -12094,44 +12166,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1164-FAKE16-NEXT: s_not_b32 s10, s2
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1164-FAKE16-NEXT: s_not_b32 s11, s6
+; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1164-FAKE16-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1164-FAKE16-NEXT: v_add_f16_e32 v0, s10, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -12148,14 +12221,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1132-TRUE16-NEXT: s_not_b32 s10, s6
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -12164,7 +12238,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
@@ -12194,14 +12268,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1132-FAKE16-NEXT: s_not_b32 s10, s6
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12211,7 +12286,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -12233,30 +12308,31 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1264-TRUE16-NEXT: s_not_b32 s10, s2
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1264-TRUE16-NEXT: s_not_b32 s11, s6
+; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
+; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s10, v0.l
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
@@ -12264,13 +12340,13 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -12280,44 +12356,45 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s9, s2, 3
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s9
-; GFX1264-FAKE16-NEXT: s_not_b32 s10, s2
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1264-FAKE16-NEXT: s_not_b32 s11, s6
+; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1264-FAKE16-NEXT: v_add_f16_e32 v0, s8, v0
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
+; GFX1264-FAKE16-NEXT: v_add_f16_e32 v0, s10, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -12334,14 +12411,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-TRUE16-NEXT: s_not_b32 s3, s3
-; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1232-TRUE16-NEXT: s_not_b32 s10, s6
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -12350,7 +12428,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -12380,14 +12458,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-FAKE16-NEXT: s_not_b32 s3, s3
-; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s2
+; GFX1232-FAKE16-NEXT: s_not_b32 s10, s6
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12397,7 +12476,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -12432,14 +12511,15 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
-; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s6
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
@@ -12475,16 +12555,17 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
-; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s3, s2, 3
-; GFX8-NEXT: s_lshl_b32 s10, s3, 3
-; GFX8-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s7, s2, 3
+; GFX8-NEXT: s_lshl_b32 s10, s7, 3
; GFX8-NEXT: s_lshl_b32 s2, s6, 16
-; GFX8-NEXT: s_not_b32 s3, s3
+; GFX8-NEXT: s_lshl_b32 s6, 0xffff, s10
+; GFX8-NEXT: s_not_b32 s11, s6
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v0, s10
@@ -12496,7 +12577,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, s3, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s11, v1
; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
@@ -12521,31 +12602,32 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[8:9], 0
-; GFX9-NEXT: s_movk_i32 s11, 0x7fff
+; GFX9-NEXT: s_movk_i32 s12, 0x7fff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s3, s2, 3
-; GFX9-NEXT: s_lshl_b32 s10, s3, 3
-; GFX9-NEXT: s_lshl_b32 s3, 0xffff, s10
+; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s7, s2, 3
+; GFX9-NEXT: s_lshl_b32 s10, s7, 3
; GFX9-NEXT: s_lshl_b32 s2, s6, 16
-; GFX9-NEXT: s_not_b32 s3, s3
+; GFX9-NEXT: s_lshl_b32 s6, 0xffff, s10
+; GFX9-NEXT: s_not_b32 s11, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_f32_e32 v0, s2, v0
; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX9-NEXT: v_add3_u32 v2, v2, v0, s11
+; GFX9-NEXT: v_add3_u32 v2, v2, v0, s12
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_and_or_b32 v0, v1, s3, v0
+; GFX9-NEXT: v_and_or_b32 v0, v1, s11, v0
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -12568,43 +12650,44 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[8:9], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_and_b32 s4, s2, -4
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
-; GFX1064-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX1064-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1064-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1064-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1064-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_not_b32 s9, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_not_b32 s10, s7
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX1064-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX1064-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1064-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1064-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX1064-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1064-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v2
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_cbranch_execnz .LBB19_1
; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -12615,42 +12698,43 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s8, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s2, -4
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
-; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
+; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1032-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1032-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_not_b32 s8, s3
-; GFX1032-NEXT: s_mov_b32 s3, 0
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_not_b32 s9, s7
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX1032-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1032-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1032-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1032-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1032-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1032-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1032-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v2
-; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_cbranch_execnz .LBB19_1
; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: s_mov_b32 s2, -1
@@ -12662,28 +12746,29 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1164-TRUE16-NEXT: s_not_b32 s9, s2
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-TRUE16-NEXT: s_not_b32 s10, s7
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: .p2align 6
; GFX1164-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1164-TRUE16-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1164-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX1164-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1164-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -12693,9 +12778,9 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
+; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
@@ -12703,13 +12788,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -12720,28 +12805,29 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1164-FAKE16-NEXT: s_not_b32 s9, s2
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-FAKE16-NEXT: s_not_b32 s10, s7
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: .p2align 6
; GFX1164-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1164-FAKE16-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX1164-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -12750,9 +12836,9 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
@@ -12760,13 +12846,13 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
@@ -12777,20 +12863,21 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1132-TRUE16-NEXT: s_not_b32 s8, s3
-; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-TRUE16-NEXT: s_not_b32 s9, s7
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: .p2align 6
; GFX1132-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12798,7 +12885,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1132-TRUE16-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1132-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1132-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1132-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -12810,18 +12897,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1132-TRUE16-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1132-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1132-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -12833,20 +12920,21 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1132-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1132-FAKE16-NEXT: s_not_b32 s8, s3
-; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1132-FAKE16-NEXT: s_not_b32 s9, s7
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: .p2align 6
; GFX1132-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -12854,7 +12942,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1132-FAKE16-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1132-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1132-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1132-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -12865,18 +12953,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1132-FAKE16-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1132-FAKE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1132-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -1
@@ -12888,27 +12976,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1264-TRUE16-NEXT: s_not_b32 s9, s2
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-TRUE16-NEXT: s_not_b32 s10, s7
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1264-TRUE16-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1264-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1264-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -12919,22 +13008,22 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v2
+; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -12945,27 +13034,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], 0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s8, s2, 3
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s6, 16
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s8
+; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s6, 16
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1264-FAKE16-NEXT: s_not_b32 s9, s2
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
-; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-FAKE16-NEXT: s_not_b32 s10, s7
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v1
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1264-FAKE16-NEXT: v_add_f32_e32 v0, s11, v0
; GFX1264-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1264-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
@@ -12975,22 +13065,22 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1264-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s8, v0
+; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v0
+; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-FAKE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -13001,27 +13091,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1232-TRUE16-NEXT: s_not_b32 s8, s3
-; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-TRUE16-NEXT: s_not_b32 s9, s7
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1232-TRUE16-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1232-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1232-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -13034,18 +13125,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1232-TRUE16-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1232-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1232-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -13057,27 +13148,28 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1232-FAKE16-NEXT: s_mov_b32 s8, 0
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s9, s6, 16
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 16
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s7, 0xffff, s2
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX1232-FAKE16-NEXT: s_not_b32 s8, s3
-; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s7
+; GFX1232-FAKE16-NEXT: s_not_b32 s9, s7
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_add_f32_e32 v0, s9, v0
+; GFX1232-FAKE16-NEXT: v_add_f32_e32 v0, s10, v0
; GFX1232-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1232-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -13089,18 +13181,18 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v0
; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX1232-FAKE16-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1232-FAKE16-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB19_1
; GFX1232-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v2
; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
@@ -13210,25 +13302,25 @@ define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addr
; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_pk_add_f16 v0, v1, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX9-NEXT: v_pk_add_f16 v1, v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_cbranch_execnz .LBB20_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: uniform_fadd_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 8863244c1e91b..f39c0a911a3f7 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -8851,10 +8851,10 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
; GFX950-NEXT: global_load_ushort v4, v[2:3], off offset:12
; GFX950-NEXT: global_load_ushort v5, v[2:3], off offset:8
; GFX950-NEXT: global_load_ushort v6, v[2:3], off offset:4
-; GFX950-NEXT: global_load_ushort v7, v[2:3], off
-; GFX950-NEXT: global_load_ushort v8, v[2:3], off offset:6
-; GFX950-NEXT: global_load_ushort v9, v[2:3], off offset:10
-; GFX950-NEXT: global_load_ushort v10, v[2:3], off offset:14
+; GFX950-NEXT: global_load_ushort v9, v[2:3], off
+; GFX950-NEXT: global_load_ushort v10, v[2:3], off offset:6
+; GFX950-NEXT: global_load_ushort v8, v[2:3], off offset:10
+; GFX950-NEXT: global_load_ushort v7, v[2:3], off offset:14
; GFX950-NEXT: global_load_ushort v11, v[2:3], off offset:18
; GFX950-NEXT: global_load_ushort v12, v[2:3], off offset:28
; GFX950-NEXT: global_load_ushort v13, v[2:3], off offset:24
@@ -8881,63 +8881,74 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
; GFX950-NEXT: global_load_ushort v58, v[2:3], off offset:58
; GFX950-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX950-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
; GFX950-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
+; GFX950-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
; GFX950-NEXT: s_waitcnt vmcnt(31)
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX950-NEXT: s_waitcnt vmcnt(30)
-; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v4
+; GFX950-NEXT: v_lshlrev_b32_e32 v29, 16, v4
; GFX950-NEXT: s_waitcnt vmcnt(29)
-; GFX950-NEXT: v_lshlrev_b32_e32 v28, 16, v5
+; GFX950-NEXT: v_lshlrev_b32_e32 v27, 16, v5
; GFX950-NEXT: v_cvt_f64_f32_e32 v[4:5], v1
; GFX950-NEXT: s_waitcnt vmcnt(27)
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v9
; GFX950-NEXT: s_waitcnt vmcnt(26)
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX950-NEXT: v_lshlrev_b32_e32 v9, 16, v10
; GFX950-NEXT: s_waitcnt vmcnt(25)
-; GFX950-NEXT: v_lshlrev_b32_e32 v27, 16, v9
+; GFX950-NEXT: v_lshlrev_b32_e32 v10, 16, v8
; GFX950-NEXT: s_waitcnt vmcnt(24)
-; GFX950-NEXT: v_lshlrev_b32_e32 v29, 16, v10
+; GFX950-NEXT: v_lshlrev_b32_e32 v28, 16, v7
; GFX950-NEXT: s_waitcnt vmcnt(23)
-; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v11
+; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v11
; GFX950-NEXT: s_waitcnt vmcnt(22)
-; GFX950-NEXT: v_lshlrev_b32_e32 v38, 16, v12
+; GFX950-NEXT: v_lshlrev_b32_e32 v37, 16, v12
; GFX950-NEXT: s_waitcnt vmcnt(21)
-; GFX950-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX950-NEXT: v_lshlrev_b32_e32 v35, 16, v13
; GFX950-NEXT: s_waitcnt vmcnt(20)
-; GFX950-NEXT: v_lshlrev_b32_e32 v34, 16, v14
+; GFX950-NEXT: v_lshlrev_b32_e32 v33, 16, v14
; GFX950-NEXT: s_waitcnt vmcnt(19)
-; GFX950-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v15
; GFX950-NEXT: s_waitcnt vmcnt(18)
-; GFX950-NEXT: v_lshlrev_b32_e32 v33, 16, v16
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[12:13], v27
+; GFX950-NEXT: v_lshlrev_b32_e32 v32, 16, v16
+; GFX950-NEXT: s_waitcnt vmcnt(17)
+; GFX950-NEXT: v_lshlrev_b32_e32 v34, 16, v17
; GFX950-NEXT: s_waitcnt vmcnt(16)
-; GFX950-NEXT: v_lshlrev_b32_e32 v37, 16, v18
+; GFX950-NEXT: v_lshlrev_b32_e32 v36, 16, v18
; GFX950-NEXT: s_waitcnt vmcnt(15)
-; GFX950-NEXT: v_lshlrev_b32_e32 v39, 16, v19
+; GFX950-NEXT: v_lshlrev_b32_e32 v38, 16, v19
; GFX950-NEXT: s_waitcnt vmcnt(14)
; GFX950-NEXT: v_lshlrev_b32_e32 v44, 16, v20
; GFX950-NEXT: s_waitcnt vmcnt(13)
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v21
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[14:15], v30
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[20:21], v31
+; GFX950-NEXT: v_lshlrev_b32_e32 v53, 16, v21
+; GFX950-NEXT: s_waitcnt vmcnt(12)
+; GFX950-NEXT: v_lshlrev_b32_e32 v49, 16, v22
+; GFX950-NEXT: s_waitcnt vmcnt(11)
+; GFX950-NEXT: v_lshlrev_b32_e32 v39, 16, v23
; GFX950-NEXT: s_waitcnt vmcnt(10)
-; GFX950-NEXT: v_lshlrev_b32_e32 v49, 16, v24
+; GFX950-NEXT: v_lshlrev_b32_e32 v48, 16, v24
; GFX950-NEXT: s_waitcnt vmcnt(9)
-; GFX950-NEXT: v_lshlrev_b32_e32 v53, 16, v25
+; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v25
; GFX950-NEXT: s_waitcnt vmcnt(8)
-; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v26
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v26
; GFX950-NEXT: s_waitcnt vmcnt(7)
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v42
; GFX950-NEXT: s_waitcnt vmcnt(6)
; GFX950-NEXT: v_lshlrev_b32_e32 v42, 16, v43
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[18:19], v32
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[24:25], v33
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[26:27], v36
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[32:33], v37
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[30:31], v38
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[36:37], v39
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[12:13], v10
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[10:11], v27
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[16:17], v28
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[14:15], v29
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[20:21], v30
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[18:19], v31
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[24:25], v32
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[22:23], v33
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[28:29], v34
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[26:27], v35
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[32:33], v36
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[30:31], v37
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[36:37], v38
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[34:35], v39
; GFX950-NEXT: v_cvt_f64_f32_e32 v[38:39], v44
; GFX950-NEXT: v_cvt_f64_f32_e32 v[44:45], v42
; GFX950-NEXT: s_waitcnt vmcnt(5)
@@ -8953,27 +8964,19 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
; GFX950-NEXT: v_cvt_f64_f32_e32 v[42:43], v1
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v57
; GFX950-NEXT: v_cvt_f64_f32_e32 v[56:57], v46
-; GFX950-NEXT: v_lshlrev_b32_e32 v35, 16, v17
-; GFX950-NEXT: v_lshlrev_b32_e32 v48, 16, v23
-; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v22
; GFX950-NEXT: scratch_store_dwordx4 v0, v[56:59], off offset:224
; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[10:11], v28
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[40:41], v40
; GFX950-NEXT: v_cvt_f64_f32_e32 v[58:59], v1
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v60
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[16:17], v29
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[22:23], v34
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[28:29], v35
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[34:35], v48
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[50:51], v49
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[48:49], v52
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[54:55], v53
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[52:53], v40
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[40:41], v41
; GFX950-NEXT: v_cvt_f64_f32_e32 v[56:57], v1
; GFX950-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[8:9], v7
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[8:9], v9
; GFX950-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[50:51], v48
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[48:49], v49
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[54:55], v52
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[52:53], v53
; GFX950-NEXT: scratch_store_dwordx4 v0, v[56:59], off offset:208
; GFX950-NEXT: scratch_store_dwordx4 v0, v[42:45], off offset:192
; GFX950-NEXT: scratch_store_dwordx4 v0, v[38:41], off offset:176
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index e2f9160438581..d907ec0341341 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -240,18 +240,18 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
+; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
-; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
+; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
; SDAG-GFX942-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -268,10 +268,10 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x2000
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -315,23 +315,23 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; SDAG-GFX1100-NEXT: s_load_b128 s[8:11], s[4:5], 0x44
; SDAG-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x34
; SDAG-GFX1100-NEXT: s_load_b32 s15, s[4:5], 0x54
+; SDAG-GFX1100-NEXT: s_mov_b32 s16, 0
; SDAG-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s4, s1
; SDAG-GFX1100-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s12, s9
; SDAG-GFX1100-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX1100-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
; SDAG-GFX1100-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s2
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX1100-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s2
-; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s1, 0x2000
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s1
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX1100-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s1
+; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s16, 0x2000
; SDAG-GFX1100-NEXT: s_clause 0xf
; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v60, s[4:7], 0 offen
; SDAG-GFX1100-NEXT: buffer_load_b128 v[4:7], v60, s[4:7], 0 offen offset:16
@@ -388,25 +388,25 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-LABEL: memcpy_known:
; GISEL-GFX942: ; %bb.0:
; GISEL-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
-; GISEL-GFX942-NEXT: s_load_dword s11, s[4:5], 0x34
-; GISEL-GFX942-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; GISEL-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
; GISEL-GFX942-NEXT: s_mov_b32 s16, 0
; GISEL-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX942-NEXT: s_mov_b32 s8, s1
-; GISEL-GFX942-NEXT: s_mov_b32 s9, s2
-; GISEL-GFX942-NEXT: s_mov_b32 s10, s3
-; GISEL-GFX942-NEXT: s_mov_b32 s4, s13
-; GISEL-GFX942-NEXT: s_mov_b32 s5, s14
-; GISEL-GFX942-NEXT: s_mov_b32 s6, s15
+; GISEL-GFX942-NEXT: s_mov_b32 s12, s9
+; GISEL-GFX942-NEXT: s_mov_b32 s4, s1
+; GISEL-GFX942-NEXT: s_mov_b32 s5, s2
+; GISEL-GFX942-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX942-NEXT: s_mov_b32 s13, s10
+; GISEL-GFX942-NEXT: s_mov_b32 s14, s11
; GISEL-GFX942-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
-; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: s_add_i32 s2, s8, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
@@ -415,55 +415,55 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v12 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:48
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[8:11], 0 offen offset:64
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[8:11], 0 offen offset:80
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[8:11], 0 offen offset:96
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[8:11], 0 offen offset:112
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[8:11], 0 offen offset:128
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[8:11], 0 offen offset:144
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[8:11], 0 offen offset:160
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[8:11], 0 offen offset:176
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[8:11], 0 offen offset:192
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[8:11], 0 offen offset:208
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[4:7], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[4:7], 0 offen offset:240
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[12:15], 0 offen
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[12:15], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen offset:32
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[12:15], 0 offen offset:48
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[12:15], 0 offen offset:64
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[12:15], 0 offen offset:80
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[12:15], 0 offen offset:96
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[12:15], 0 offen offset:112
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[12:15], 0 offen offset:128
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[12:15], 0 offen offset:144
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[12:15], 0 offen offset:160
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[12:15], 0 offen offset:176
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[12:15], 0 offen offset:192
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[12:15], 0 offen offset:208
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[4:7], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[12:15], 0 offen offset:224
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[4:7], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[12:15], 0 offen offset:240
; GISEL-GFX942-NEXT: s_cbranch_scc1 .LBB0_1
; GISEL-GFX942-NEXT: ; %bb.2: ; %static-memcpy-post-expansion
; GISEL-GFX942-NEXT: s_endpgm
@@ -761,18 +761,18 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
; SDAG-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
; SDAG-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
+; SDAG-GFX942-NEXT: s_mov_b32 s16, 0
; SDAG-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
+; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
; SDAG-GFX942-NEXT: s_mov_b32 s4, s1
; SDAG-GFX942-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
-; SDAG-GFX942-NEXT: s_mov_b32 s12, s9
+; SDAG-GFX942-NEXT: s_mov_b32 s6, s3
; SDAG-GFX942-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX942-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX942-NEXT: s_mov_b32 s14, s11
; SDAG-GFX942-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX942-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX942-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v0, s[4:7], 0 offen
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v0, s[4:7], 0 offen offset:16
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v0, s[4:7], 0 offen offset:32
@@ -789,10 +789,10 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v0, s[4:7], 0 offen offset:208
; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v0, s[4:7], 0 offen offset:224
; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v0, s[4:7], 0 offen offset:240
-; SDAG-GFX942-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX942-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s2
-; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s1, 0x100
+; SDAG-GFX942-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX942-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX942-NEXT: v_mov_b32_e32 v0, s1
+; SDAG-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen
; SDAG-GFX942-NEXT: s_waitcnt vmcnt(15)
@@ -836,23 +836,23 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; SDAG-GFX1100-NEXT: s_load_b128 s[8:11], s[4:5], 0x44
; SDAG-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x34
; SDAG-GFX1100-NEXT: s_load_b32 s15, s[4:5], 0x54
+; SDAG-GFX1100-NEXT: s_mov_b32 s16, 0
; SDAG-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s4, s1
; SDAG-GFX1100-NEXT: s_mov_b32 s5, s2
-; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
+; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
; SDAG-GFX1100-NEXT: s_mov_b32 s12, s9
; SDAG-GFX1100-NEXT: s_mov_b32 s13, s10
-; SDAG-GFX1100-NEXT: s_mov_b32 s1, 0
+; SDAG-GFX1100-NEXT: s_mov_b32 s14, s11
; SDAG-GFX1100-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; SDAG-GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s0, s1
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s2
-; SDAG-GFX1100-NEXT: s_add_i32 s2, s8, s1
-; SDAG-GFX1100-NEXT: s_addk_i32 s1, 0x100
-; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s2
-; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s1, 0x100
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s0, s16
+; SDAG-GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v60, s1
+; SDAG-GFX1100-NEXT: s_add_i32 s1, s8, s16
+; SDAG-GFX1100-NEXT: s_addk_i32 s16, 0x100
+; SDAG-GFX1100-NEXT: v_mov_b32_e32 v64, s1
+; SDAG-GFX1100-NEXT: s_cmpk_lt_u32 s16, 0x100
; SDAG-GFX1100-NEXT: s_clause 0xf
; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v60, s[4:7], 0 offen
; SDAG-GFX1100-NEXT: buffer_load_b128 v[4:7], v60, s[4:7], 0 offen offset:16
@@ -909,25 +909,25 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-LABEL: memcpy_known_medium:
; GISEL-GFX942: ; %bb.0:
; GISEL-GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x54
-; GISEL-GFX942-NEXT: s_load_dword s11, s[4:5], 0x34
-; GISEL-GFX942-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s7, s[4:5], 0x34
+; GISEL-GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x44
+; GISEL-GFX942-NEXT: s_load_dword s15, s[4:5], 0x54
; GISEL-GFX942-NEXT: s_mov_b32 s16, 0
; GISEL-GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX942-NEXT: s_mov_b32 s8, s1
-; GISEL-GFX942-NEXT: s_mov_b32 s9, s2
-; GISEL-GFX942-NEXT: s_mov_b32 s10, s3
-; GISEL-GFX942-NEXT: s_mov_b32 s4, s13
-; GISEL-GFX942-NEXT: s_mov_b32 s5, s14
-; GISEL-GFX942-NEXT: s_mov_b32 s6, s15
+; GISEL-GFX942-NEXT: s_mov_b32 s12, s9
+; GISEL-GFX942-NEXT: s_mov_b32 s4, s1
+; GISEL-GFX942-NEXT: s_mov_b32 s5, s2
+; GISEL-GFX942-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX942-NEXT: s_mov_b32 s13, s10
+; GISEL-GFX942-NEXT: s_mov_b32 s14, s11
; GISEL-GFX942-NEXT: .LBB1_1: ; %static-memcpy-expansion-main-body
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
-; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: s_add_i32 s2, s8, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
@@ -936,55 +936,55 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v12 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:48
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[8:11], 0 offen offset:64
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[8:11], 0 offen offset:80
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[8:11], 0 offen offset:96
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[8:11], 0 offen offset:112
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[8:11], 0 offen offset:128
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[8:11], 0 offen offset:144
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[8:11], 0 offen offset:160
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[8:11], 0 offen offset:176
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[8:11], 0 offen offset:192
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[8:11], 0 offen offset:208
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[16:19], v60, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[20:23], v60, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[24:27], v60, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[28:31], v60, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[32:35], v60, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[36:39], v60, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[40:43], v60, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[44:47], v60, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[48:51], v60, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[52:55], v60, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[4:7], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[4:7], 0 offen offset:240
; GISEL-GFX942-NEXT: s_nop 0
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[12:15], 0 offen
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[12:15], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[4:7], 0 offen offset:32
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v0, s[12:15], 0 offen offset:32
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[4:7], 0 offen offset:48
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[12:15], v0, s[12:15], 0 offen offset:48
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[4:7], 0 offen offset:64
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[16:19], v0, s[12:15], 0 offen offset:64
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[4:7], 0 offen offset:80
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[20:23], v0, s[12:15], 0 offen offset:80
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[4:7], 0 offen offset:96
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[24:27], v0, s[12:15], 0 offen offset:96
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[4:7], 0 offen offset:112
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[28:31], v0, s[12:15], 0 offen offset:112
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[4:7], 0 offen offset:128
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[32:35], v0, s[12:15], 0 offen offset:128
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[4:7], 0 offen offset:144
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[36:39], v0, s[12:15], 0 offen offset:144
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[4:7], 0 offen offset:160
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[40:43], v0, s[12:15], 0 offen offset:160
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[4:7], 0 offen offset:176
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[44:47], v0, s[12:15], 0 offen offset:176
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[4:7], 0 offen offset:192
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[48:51], v0, s[12:15], 0 offen offset:192
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[4:7], 0 offen offset:208
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[52:55], v0, s[12:15], 0 offen offset:208
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[4:7], 0 offen offset:224
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[56:59], v0, s[12:15], 0 offen offset:224
; GISEL-GFX942-NEXT: s_waitcnt vmcnt(15)
-; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[4:7], 0 offen offset:240
+; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[60:63], v0, s[12:15], 0 offen offset:240
; GISEL-GFX942-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-GFX942-NEXT: ; %bb.2: ; %static-memcpy-post-expansion
; GISEL-GFX942-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 293e24f2d1b9d..0137b14932757 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -1401,11 +1401,12 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
; VI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; VI-NEXT: s_mov_b32 s38, -1
; VI-NEXT: s_mov_b32 s39, 0xe80000
-; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s1, s0
; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_addc_u32 s37, s37, 0
; VI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1423,11 +1424,12 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
; CI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; CI-NEXT: s_mov_b32 s38, -1
; CI-NEXT: s_mov_b32 s39, 0xe8f000
-; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; CI-NEXT: s_mov_b64 s[0:1], 0
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; CI-NEXT: s_addc_u32 s37, s37, 0
; CI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1445,11 +1447,12 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
; SDAG-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; SDAG-NEXT: s_mov_b32 s38, -1
; SDAG-NEXT: s_mov_b32 s39, 0xe00000
-; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b64 s[6:7], s[0:1]
-; SDAG-NEXT: s_mov_b64 s[0:1], 0
+; SDAG-NEXT: s_mov_b32 s0, 0
+; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b32 s3, 0xf000
; SDAG-NEXT: s_mov_b32 s2, -1
+; SDAG-NEXT: s_mov_b32 s1, s0
; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; SDAG-NEXT: s_addc_u32 s37, s37, 0
; SDAG-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1463,9 +1466,10 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
;
; GFX11-LABEL: test_call_external_void_func_v2i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b64 s[4:5], 0
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: s_getpc_b64 s[2:3]
; GFX11-NEXT: s_add_u32 s2, s2, external_void_func_v2i64 at rel32@lo+4
; GFX11-NEXT: s_addc_u32 s3, s3, external_void_func_v2i64 at rel32@hi+12
@@ -1479,10 +1483,11 @@ define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
; HSA: ; %bb.0:
; HSA-NEXT: s_add_i32 s6, s6, s9
; HSA-NEXT: s_lshr_b32 flat_scratch_hi, s6, 8
+; HSA-NEXT: s_mov_b32 s8, 0
; HSA-NEXT: s_add_u32 s0, s0, s9
-; HSA-NEXT: s_mov_b64 s[8:9], 0
; HSA-NEXT: s_mov_b32 s11, 0x1100f000
; HSA-NEXT: s_mov_b32 s10, -1
+; HSA-NEXT: s_mov_b32 s9, s8
; HSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; HSA-NEXT: s_addc_u32 s1, s1, 0
; HSA-NEXT: s_mov_b32 flat_scratch_lo, s7
@@ -1653,11 +1658,12 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
; VI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; VI-NEXT: s_mov_b32 s38, -1
; VI-NEXT: s_mov_b32 s39, 0xe80000
-; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s1, s0
; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_addc_u32 s37, s37, 0
; VI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1677,11 +1683,12 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
; CI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; CI-NEXT: s_mov_b32 s38, -1
; CI-NEXT: s_mov_b32 s39, 0xe8f000
-; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; CI-NEXT: s_mov_b64 s[0:1], 0
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; CI-NEXT: s_addc_u32 s37, s37, 0
; CI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1701,11 +1708,12 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
; SDAG-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; SDAG-NEXT: s_mov_b32 s38, -1
; SDAG-NEXT: s_mov_b32 s39, 0xe00000
-; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b64 s[6:7], s[0:1]
-; SDAG-NEXT: s_mov_b64 s[0:1], 0
+; SDAG-NEXT: s_mov_b32 s0, 0
+; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b32 s3, 0xf000
; SDAG-NEXT: s_mov_b32 s2, -1
+; SDAG-NEXT: s_mov_b32 s1, s0
; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; SDAG-NEXT: s_addc_u32 s37, s37, 0
; SDAG-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1721,9 +1729,10 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
;
; GFX11-LABEL: test_call_external_void_func_v3i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b64 s[4:5], 0
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 2
; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
; GFX11-NEXT: s_getpc_b64 s[2:3]
@@ -1738,10 +1747,11 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
; HSA: ; %bb.0:
; HSA-NEXT: s_add_i32 s6, s6, s9
; HSA-NEXT: s_lshr_b32 flat_scratch_hi, s6, 8
+; HSA-NEXT: s_mov_b32 s8, 0
; HSA-NEXT: s_add_u32 s0, s0, s9
-; HSA-NEXT: s_mov_b64 s[8:9], 0
; HSA-NEXT: s_mov_b32 s11, 0x1100f000
; HSA-NEXT: s_mov_b32 s10, -1
+; HSA-NEXT: s_mov_b32 s9, s8
; HSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; HSA-NEXT: s_addc_u32 s1, s1, 0
; HSA-NEXT: s_mov_b32 flat_scratch_lo, s7
@@ -1795,11 +1805,12 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
; VI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; VI-NEXT: s_mov_b32 s38, -1
; VI-NEXT: s_mov_b32 s39, 0xe80000
-; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_add_u32 s36, s36, s3
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s1, s0
; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_addc_u32 s37, s37, 0
; VI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1821,11 +1832,12 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
; CI-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; CI-NEXT: s_mov_b32 s38, -1
; CI-NEXT: s_mov_b32 s39, 0xe8f000
-; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b64 s[6:7], s[0:1]
-; CI-NEXT: s_mov_b64 s[0:1], 0
+; CI-NEXT: s_mov_b32 s0, 0
+; CI-NEXT: s_add_u32 s36, s36, s3
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; CI-NEXT: s_addc_u32 s37, s37, 0
; CI-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1847,11 +1859,12 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
; SDAG-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1
; SDAG-NEXT: s_mov_b32 s38, -1
; SDAG-NEXT: s_mov_b32 s39, 0xe00000
-; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b64 s[6:7], s[0:1]
-; SDAG-NEXT: s_mov_b64 s[0:1], 0
+; SDAG-NEXT: s_mov_b32 s0, 0
+; SDAG-NEXT: s_add_u32 s36, s36, s3
; SDAG-NEXT: s_mov_b32 s3, 0xf000
; SDAG-NEXT: s_mov_b32 s2, -1
+; SDAG-NEXT: s_mov_b32 s1, s0
; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
; SDAG-NEXT: s_addc_u32 s37, s37, 0
; SDAG-NEXT: s_mov_b64 s[0:1], s[36:37]
@@ -1869,9 +1882,10 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
;
; GFX11-LABEL: test_call_external_void_func_v4i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b64 s[4:5], 0
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: v_dual_mov_b32 v4, 1 :: v_dual_mov_b32 v5, 2
; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
; GFX11-NEXT: v_dual_mov_b32 v6, 3 :: v_dual_mov_b32 v7, 4
@@ -1887,10 +1901,11 @@ define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
; HSA: ; %bb.0:
; HSA-NEXT: s_add_i32 s6, s6, s9
; HSA-NEXT: s_lshr_b32 flat_scratch_hi, s6, 8
+; HSA-NEXT: s_mov_b32 s8, 0
; HSA-NEXT: s_add_u32 s0, s0, s9
-; HSA-NEXT: s_mov_b64 s[8:9], 0
; HSA-NEXT: s_mov_b32 s11, 0x1100f000
; HSA-NEXT: s_mov_b32 s10, -1
+; HSA-NEXT: s_mov_b32 s9, s8
; HSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; HSA-NEXT: s_addc_u32 s1, s1, 0
; HSA-NEXT: s_mov_b32 flat_scratch_lo, s7
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index 77ee1ada2af94..5637f429cf0f8 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -1452,17 +1452,18 @@ entry:
define amdgpu_kernel void @amd_kernel_v2i8(<2 x i8> %arg0) {
; SI-LABEL: amd_kernel_v2i8:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bfe_u32 s4, s2, 0x80008
+; SI-NEXT: s_bfe_u32 s2, s1, 0x80008
+; SI-NEXT: s_add_i32 s1, s1, s1
+; SI-NEXT: s_and_b32 s1, s1, 0xff
; SI-NEXT: s_add_i32 s2, s2, s2
-; SI-NEXT: s_and_b32 s2, s2, 0xff
-; SI-NEXT: s_add_i32 s4, s4, s4
-; SI-NEXT: s_lshl_b32 s4, s4, 8
-; SI-NEXT: s_or_b32 s4, s2, s4
+; SI-NEXT: s_lshl_b32 s2, s2, 8
+; SI-NEXT: s_or_b32 s4, s1, s2
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1524,27 +1525,28 @@ entry:
define amdgpu_kernel void @amd_kernel_v4i8(<4 x i8> %arg0) {
; SI-LABEL: amd_kernel_v4i8:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s4, s2, 16
-; SI-NEXT: s_lshr_b32 s5, s2, 24
-; SI-NEXT: s_bfe_u32 s6, s2, 0x80008
+; SI-NEXT: s_lshr_b32 s2, s1, 16
+; SI-NEXT: s_lshr_b32 s4, s1, 24
+; SI-NEXT: s_bfe_u32 s5, s1, 0x80008
+; SI-NEXT: s_add_i32 s1, s1, s1
+; SI-NEXT: s_add_i32 s4, s4, s4
; SI-NEXT: s_add_i32 s2, s2, s2
+; SI-NEXT: s_and_b32 s1, s1, 0xff
; SI-NEXT: s_add_i32 s5, s5, s5
-; SI-NEXT: s_add_i32 s4, s4, s4
+; SI-NEXT: s_lshl_b32 s4, s4, 24
; SI-NEXT: s_and_b32 s2, s2, 0xff
-; SI-NEXT: s_add_i32 s6, s6, s6
-; SI-NEXT: s_lshl_b32 s5, s5, 24
-; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 8
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_or_b32 s2, s2, s6
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_and_b32 s2, s2, 0xffff
-; SI-NEXT: s_or_b32 s4, s2, s4
+; SI-NEXT: s_lshl_b32 s5, s5, 8
+; SI-NEXT: s_lshl_b32 s2, s2, 16
+; SI-NEXT: s_or_b32 s1, s1, s5
+; SI-NEXT: s_or_b32 s2, s4, s2
+; SI-NEXT: s_and_b32 s1, s1, 0xffff
+; SI-NEXT: s_or_b32 s4, s1, s2
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1637,7 +1639,8 @@ define amdgpu_kernel void @amd_kernel_v3i8(<3 x i8> %arg0) {
; SI-LABEL: amd_kernel_v3i8:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dword s4, s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[0:1], 2
+; SI-NEXT: s_mov_b32 s1, 0
+; SI-NEXT: s_mov_b32 s0, 2
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
@@ -1649,9 +1652,9 @@ define amdgpu_kernel void @amd_kernel_v3i8(<3 x i8> %arg0) {
; SI-NEXT: s_add_i32 s5, s5, s5
; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s0, s1
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -1731,7 +1734,8 @@ define amdgpu_kernel void @amd_kernel_v5i8(<5 x i8> %arg0) {
; SI-LABEL: amd_kernel_v5i8:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[0:1], 4
+; SI-NEXT: s_mov_b32 s1, 0
+; SI-NEXT: s_mov_b32 s0, 4
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
@@ -1745,16 +1749,16 @@ define amdgpu_kernel void @amd_kernel_v5i8(<5 x i8> %arg0) {
; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_add_i32 s8, s8, s8
; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
-; SI-NEXT: s_lshl_b32 s0, s7, 24
-; SI-NEXT: s_and_b32 s1, s6, 0xff
-; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: s_lshl_b32 s1, s1, 16
+; SI-NEXT: s_lshl_b32 s5, s7, 24
+; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s7, s8, 8
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s4, s4, s7
+; SI-NEXT: s_or_b32 s5, s5, s6
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_or_b32 s0, s0, s1
-; SI-NEXT: s_and_b32 s1, s4, 0xffff
-; SI-NEXT: s_or_b32 s4, s1, s0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s0, s1
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1861,44 +1865,45 @@ define amdgpu_kernel void @amd_kernel_v8i8(<8 x i8> %arg0) {
; SI-LABEL: amd_kernel_v8i8:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s2, s4, 16
-; SI-NEXT: s_lshr_b32 s6, s4, 24
-; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: s_lshr_b32 s8, s5, 24
-; SI-NEXT: s_bfe_u32 s9, s4, 0x80008
-; SI-NEXT: s_bfe_u32 s10, s5, 0x80008
+; SI-NEXT: s_lshr_b32 s1, s4, 16
+; SI-NEXT: s_lshr_b32 s2, s4, 24
+; SI-NEXT: s_lshr_b32 s6, s5, 16
+; SI-NEXT: s_lshr_b32 s7, s5, 24
+; SI-NEXT: s_bfe_u32 s8, s4, 0x80008
+; SI-NEXT: s_bfe_u32 s9, s5, 0x80008
; SI-NEXT: s_add_i32 s5, s5, s5
; SI-NEXT: s_add_i32 s4, s4, s4
-; SI-NEXT: s_add_i32 s8, s8, s8
; SI-NEXT: s_add_i32 s7, s7, s7
-; SI-NEXT: s_and_b32 s5, s5, 0xff
-; SI-NEXT: s_add_i32 s10, s10, s10
; SI-NEXT: s_add_i32 s6, s6, s6
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: s_add_i32 s9, s9, s9
; SI-NEXT: s_add_i32 s2, s2, s2
+; SI-NEXT: s_add_i32 s1, s1, s1
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: s_add_i32 s9, s9, s9
-; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_and_b32 s7, s7, 0xff
-; SI-NEXT: s_lshl_b32 s10, s10, 8
-; SI-NEXT: s_lshl_b32 s6, s6, 24
-; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_add_i32 s8, s8, s8
+; SI-NEXT: s_lshl_b32 s7, s7, 24
+; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s9, s9, 8
-; SI-NEXT: s_lshl_b32 s7, s7, 16
-; SI-NEXT: s_or_b32 s5, s5, s10
-; SI-NEXT: s_lshl_b32 s2, s2, 16
-; SI-NEXT: s_or_b32 s4, s4, s9
-; SI-NEXT: s_or_b32 s7, s8, s7
+; SI-NEXT: s_lshl_b32 s2, s2, 24
+; SI-NEXT: s_and_b32 s1, s1, 0xff
+; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_or_b32 s5, s5, s9
+; SI-NEXT: s_lshl_b32 s1, s1, 16
+; SI-NEXT: s_or_b32 s4, s4, s8
+; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: s_or_b32 s2, s6, s2
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: s_or_b32 s4, s4, s2
+; SI-NEXT: s_or_b32 s1, s2, s1
+; SI-NEXT: s_and_b32 s2, s4, 0xffff
+; SI-NEXT: s_or_b32 s4, s5, s6
+; SI-NEXT: s_or_b32 s5, s2, s1
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: s_mov_b32 s1, s0
+; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -2039,74 +2044,75 @@ define amdgpu_kernel void @amd_kernel_v16i8(<16 x i8> %arg0) {
; SI-LABEL: amd_kernel_v16i8:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s6, s0, 16
-; SI-NEXT: s_lshr_b32 s8, s0, 24
-; SI-NEXT: s_lshr_b32 s9, s1, 16
-; SI-NEXT: s_lshr_b32 s10, s1, 24
-; SI-NEXT: s_lshr_b32 s11, s2, 16
-; SI-NEXT: s_lshr_b32 s12, s2, 24
-; SI-NEXT: s_lshr_b32 s13, s3, 16
-; SI-NEXT: s_lshr_b32 s14, s3, 24
-; SI-NEXT: s_bfe_u32 s15, s0, 0x80008
-; SI-NEXT: s_bfe_u32 s16, s1, 0x80008
-; SI-NEXT: s_bfe_u32 s17, s2, 0x80008
-; SI-NEXT: s_bfe_u32 s18, s3, 0x80008
+; SI-NEXT: s_lshr_b32 s5, s0, 16
+; SI-NEXT: s_lshr_b32 s6, s0, 24
+; SI-NEXT: s_lshr_b32 s8, s1, 16
+; SI-NEXT: s_lshr_b32 s9, s1, 24
+; SI-NEXT: s_lshr_b32 s10, s2, 16
+; SI-NEXT: s_lshr_b32 s11, s2, 24
+; SI-NEXT: s_lshr_b32 s12, s3, 16
+; SI-NEXT: s_lshr_b32 s13, s3, 24
+; SI-NEXT: s_bfe_u32 s14, s0, 0x80008
+; SI-NEXT: s_bfe_u32 s15, s1, 0x80008
+; SI-NEXT: s_bfe_u32 s16, s2, 0x80008
+; SI-NEXT: s_bfe_u32 s17, s3, 0x80008
; SI-NEXT: s_add_i32 s3, s3, s3
; SI-NEXT: s_add_i32 s2, s2, s2
; SI-NEXT: s_add_i32 s1, s1, s1
; SI-NEXT: s_add_i32 s0, s0, s0
-; SI-NEXT: s_add_i32 s14, s14, s14
; SI-NEXT: s_add_i32 s13, s13, s13
-; SI-NEXT: s_and_b32 s3, s3, 0xff
-; SI-NEXT: s_add_i32 s18, s18, s18
; SI-NEXT: s_add_i32 s12, s12, s12
-; SI-NEXT: s_add_i32 s11, s11, s11
-; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_and_b32 s3, s3, 0xff
; SI-NEXT: s_add_i32 s17, s17, s17
+; SI-NEXT: s_add_i32 s11, s11, s11
; SI-NEXT: s_add_i32 s10, s10, s10
-; SI-NEXT: s_add_i32 s9, s9, s9
-; SI-NEXT: s_and_b32 s1, s1, 0xff
+; SI-NEXT: s_and_b32 s2, s2, 0xff
; SI-NEXT: s_add_i32 s16, s16, s16
+; SI-NEXT: s_add_i32 s9, s9, s9
; SI-NEXT: s_add_i32 s8, s8, s8
+; SI-NEXT: s_and_b32 s1, s1, 0xff
+; SI-NEXT: s_add_i32 s15, s15, s15
; SI-NEXT: s_add_i32 s6, s6, s6
+; SI-NEXT: s_add_i32 s5, s5, s5
; SI-NEXT: s_and_b32 s0, s0, 0xff
-; SI-NEXT: s_add_i32 s15, s15, s15
-; SI-NEXT: s_lshl_b32 s14, s14, 24
-; SI-NEXT: s_and_b32 s13, s13, 0xff
-; SI-NEXT: s_lshl_b32 s18, s18, 8
-; SI-NEXT: s_lshl_b32 s12, s12, 24
-; SI-NEXT: s_and_b32 s11, s11, 0xff
+; SI-NEXT: s_add_i32 s14, s14, s14
+; SI-NEXT: s_lshl_b32 s13, s13, 24
+; SI-NEXT: s_and_b32 s12, s12, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 8
-; SI-NEXT: s_lshl_b32 s10, s10, 24
-; SI-NEXT: s_and_b32 s9, s9, 0xff
+; SI-NEXT: s_lshl_b32 s11, s11, 24
+; SI-NEXT: s_and_b32 s10, s10, 0xff
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: s_lshl_b32 s8, s8, 24
-; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_lshl_b32 s9, s9, 24
+; SI-NEXT: s_and_b32 s8, s8, 0xff
; SI-NEXT: s_lshl_b32 s15, s15, 8
-; SI-NEXT: s_lshl_b32 s13, s13, 16
-; SI-NEXT: s_or_b32 s3, s3, s18
-; SI-NEXT: s_lshl_b32 s11, s11, 16
-; SI-NEXT: s_or_b32 s2, s2, s17
-; SI-NEXT: s_lshl_b32 s9, s9, 16
-; SI-NEXT: s_or_b32 s1, s1, s16
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_or_b32 s0, s0, s15
-; SI-NEXT: s_or_b32 s13, s14, s13
+; SI-NEXT: s_lshl_b32 s6, s6, 24
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: s_lshl_b32 s14, s14, 8
+; SI-NEXT: s_lshl_b32 s12, s12, 16
+; SI-NEXT: s_or_b32 s3, s3, s17
+; SI-NEXT: s_lshl_b32 s10, s10, 16
+; SI-NEXT: s_or_b32 s2, s2, s16
+; SI-NEXT: s_lshl_b32 s8, s8, 16
+; SI-NEXT: s_or_b32 s1, s1, s15
+; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_or_b32 s0, s0, s14
+; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_and_b32 s3, s3, 0xffff
-; SI-NEXT: s_or_b32 s11, s12, s11
+; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_and_b32 s2, s2, 0xffff
-; SI-NEXT: s_or_b32 s9, s10, s9
+; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_and_b32 s1, s1, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s5, s6, s5
; SI-NEXT: s_and_b32 s0, s0, 0xffff
-; SI-NEXT: s_or_b32 s3, s3, s13
-; SI-NEXT: s_or_b32 s2, s2, s11
-; SI-NEXT: s_or_b32 s1, s1, s9
-; SI-NEXT: s_or_b32 s0, s0, s6
+; SI-NEXT: s_or_b32 s3, s3, s12
+; SI-NEXT: s_or_b32 s2, s2, s10
+; SI-NEXT: s_or_b32 s1, s1, s8
+; SI-NEXT: s_or_b32 s0, s0, s5
; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_mov_b32_e32 v2, s2
@@ -2352,7 +2358,8 @@ define amdgpu_kernel void @amd_kernel_v32i8(<32 x i8> %arg0) {
; SI-LABEL: amd_kernel_v32i8:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
-; SI-NEXT: s_mov_b64 s[8:9], 16
+; SI-NEXT: s_mov_b32 s9, 0
+; SI-NEXT: s_mov_b32 s8, 16
; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
@@ -2494,7 +2501,7 @@ define amdgpu_kernel void @amd_kernel_v32i8(<32 x i8> %arg0) {
; SI-NEXT: v_mov_b32_e32 v1, s1
; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: v_mov_b32_e32 v3, s3
-; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s8, s9
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
index 7a576b1c58c55..cc2f775ff22bc 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll
@@ -163,33 +163,33 @@ define amdgpu_kernel void @test_copy_v4i8_x3(ptr addrspace(1) %out0, ptr addrspa
define amdgpu_kernel void @test_copy_v4i8_x4(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3, ptr addrspace(1) %in) nounwind {
; SI-LABEL: test_copy_v4i8_x4:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x11
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s10, 0
-; SI-NEXT: s_mov_b32 s11, s3
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x11
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_mov_b32 s3, s11
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; SI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s14, s2
-; SI-NEXT: s_mov_b32 s15, s3
-; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s18, s10
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
-; SI-NEXT: s_mov_b32 s19, s3
-; SI-NEXT: s_mov_b32 s22, s2
-; SI-NEXT: s_mov_b32 s23, s3
-; SI-NEXT: s_mov_b32 s12, s6
-; SI-NEXT: s_mov_b32 s13, s7
-; SI-NEXT: s_mov_b32 s16, s8
-; SI-NEXT: s_mov_b32 s17, s9
-; SI-NEXT: s_mov_b32 s20, s10
-; SI-NEXT: s_mov_b32 s21, s11
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s19, s11
+; SI-NEXT: s_mov_b32 s22, s10
+; SI-NEXT: s_mov_b32 s23, s11
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s16, s4
+; SI-NEXT: s_mov_b32 s17, s5
+; SI-NEXT: s_mov_b32 s20, s6
+; SI-NEXT: s_mov_b32 s21, s7
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: buffer_store_dword v0, off, s[12:15], 0
; SI-NEXT: buffer_store_dword v0, off, s[16:19], 0
; SI-NEXT: buffer_store_dword v0, off, s[20:23], 0
@@ -325,23 +325,23 @@ define amdgpu_kernel void @test_copy_v4i8_extra_use(ptr addrspace(1) %out0, ptr
define amdgpu_kernel void @test_copy_v4i8_x2_extra_use(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %in) nounwind {
; SI-LABEL: test_copy_v4i8_x2_extra_use:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s14, 0
-; SI-NEXT: s_mov_b32 s15, s3
+; SI-NEXT: s_mov_b32 s15, s11
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[12:13], s[10:11]
+; SI-NEXT: s_mov_b64 s[12:13], s[6:7]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s14, s2
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
-; SI-NEXT: s_mov_b32 s12, s6
-; SI-NEXT: s_mov_b32 s13, s7
-; SI-NEXT: s_mov_b32 s10, s2
-; SI-NEXT: s_mov_b32 s11, s3
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; SI-NEXT: v_add_i32_e32 v3, vcc, 9, v0
@@ -357,9 +357,9 @@ define amdgpu_kernel void @test_copy_v4i8_x2_extra_use(ptr addrspace(1) %out0, p
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_add_i32_e32 v1, vcc, 0x9000000, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT: buffer_store_dword v1, off, s[12:15], 0
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: buffer_store_dword v1, off, s[12:15], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_copy_v4i8_x2_extra_use:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index 37f5889918c41..68b64da89a696 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -334,58 +334,58 @@ define amdgpu_kernel void @v_ctpop_v4i64(ptr addrspace(1) noalias %out, ptr addr
define amdgpu_kernel void @ctpop_i64_in_br(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %ctpop_arg, i32 %cond) {
; SI-LABEL: ctpop_i64_in_br:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
+; SI-NEXT: s_load_dword s6, s[4:5], 0xf
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s8, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: s_cbranch_scc0 .LBB7_4
; SI-NEXT: ; %bb.1: ; %else
-; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x2
+; SI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x2
; SI-NEXT: s_mov_b64 s[2:3], 0
; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 vcc, vcc
; SI-NEXT: s_cbranch_vccnz .LBB7_3
; SI-NEXT: .LBB7_2: ; %if
-; SI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
-; SI-NEXT: s_mov_b32 s5, 0
+; SI-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; SI-NEXT: s_mov_b32 s7, 0
; SI-NEXT: .LBB7_3: ; %endif
-; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: ; implicit-def: $sgpr6_sgpr7
; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: ctpop_i64_in_br:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s8, s[4:5], 0x3c
+; VI-NEXT: s_load_dword s6, s[4:5], 0x3c
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 0
; VI-NEXT: s_cbranch_scc0 .LBB7_4
; VI-NEXT: ; %bb.1: ; %else
-; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x8
+; VI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x8
; VI-NEXT: s_cbranch_execnz .LBB7_3
; VI-NEXT: .LBB7_2: ; %if
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
-; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; VI-NEXT: s_mov_b32 s7, 0
; VI-NEXT: .LBB7_3: ; %endif
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB7_4:
-; VI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; VI-NEXT: ; implicit-def: $sgpr6_sgpr7
; VI-NEXT: s_branch .LBB7_2
entry:
%tmp0 = icmp eq i32 %cond, 0
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 0eed0ba50092b..0536e97f57387 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1820,13 +1820,13 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {
; SI-LABEL: load_v7i8_to_v7f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s10, 0
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: s_mov_b64 s[8:9], s[6:7]
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: buffer_load_ubyte v2, v[0:1], s[8:11], 0 addr64 offset:3
; SI-NEXT: buffer_load_ubyte v4, v[0:1], s[8:11], 0 addr64 offset:2
@@ -1835,9 +1835,9 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; SI-NEXT: buffer_load_ubyte v7, v[0:1], s[8:11], 0 addr64 offset:5
; SI-NEXT: buffer_load_ubyte v8, v[0:1], s[8:11], 0 addr64 offset:4
; SI-NEXT: buffer_load_ubyte v9, v[0:1], s[8:11], 0 addr64 offset:6
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v3, v2
; SI-NEXT: s_waitcnt vmcnt(5)
@@ -1852,9 +1852,9 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; SI-NEXT: v_cvt_f32_ubyte0_e32 v4, v8
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v6, v9
-; SI-NEXT: buffer_store_dword v6, off, s[4:7], 0 offset:24
-; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], 0 offset:24
+; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: load_v7i8_to_v7f32:
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 0eebada5c1c9e..672803a985bf3 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -611,43 +611,43 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_ashrrev_i32_e32 v18, 31, v7
; GISEL-NEXT: v_ashrrev_i32_e32 v19, 31, v15
-; GISEL-NEXT: v_mov_b32_e32 v16, 0x7f
-; GISEL-NEXT: v_mov_b32_e32 v17, 0
+; GISEL-NEXT: v_mov_b32_e32 v10, 0x7f
+; GISEL-NEXT: v_mov_b32_e32 v11, 0
; GISEL-NEXT: v_xor_b32_e32 v0, v18, v4
; GISEL-NEXT: v_xor_b32_e32 v1, v18, v5
; GISEL-NEXT: v_xor_b32_e32 v2, v18, v6
; GISEL-NEXT: v_xor_b32_e32 v3, v18, v7
; GISEL-NEXT: v_xor_b32_e32 v4, v19, v12
; GISEL-NEXT: v_xor_b32_e32 v5, v19, v13
-; GISEL-NEXT: v_xor_b32_e32 v12, v19, v14
-; GISEL-NEXT: v_xor_b32_e32 v13, v19, v15
+; GISEL-NEXT: v_xor_b32_e32 v14, v19, v14
+; GISEL-NEXT: v_xor_b32_e32 v15, v19, v15
; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v0, v18
; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v1, v18, vcc
; GISEL-NEXT: v_sub_i32_e64 v20, s[4:5], v4, v19
; GISEL-NEXT: v_subb_u32_e64 v21, s[4:5], v5, v19, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v10, vcc, v2, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v18, vcc
-; GISEL-NEXT: v_subb_u32_e64 v4, vcc, v12, v19, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v13, v19, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v12, v21
-; GISEL-NEXT: v_ffbh_u32_e32 v13, v20
-; GISEL-NEXT: v_ffbh_u32_e32 v14, v7
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v6
+; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v2, v18, vcc
+; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v3, v18, vcc
+; GISEL-NEXT: v_subb_u32_e64 v4, vcc, v14, v19, s[4:5]
+; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v15, v19, vcc
+; GISEL-NEXT: v_ffbh_u32_e32 v14, v21
+; GISEL-NEXT: v_ffbh_u32_e32 v15, v20
+; GISEL-NEXT: v_ffbh_u32_e32 v16, v7
+; GISEL-NEXT: v_ffbh_u32_e32 v17, v6
; GISEL-NEXT: v_or_b32_e32 v0, v20, v4
; GISEL-NEXT: v_or_b32_e32 v1, v21, v5
-; GISEL-NEXT: v_or_b32_e32 v2, v6, v10
-; GISEL-NEXT: v_or_b32_e32 v3, v7, v11
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, 32, v13
+; GISEL-NEXT: v_or_b32_e32 v2, v6, v12
+; GISEL-NEXT: v_or_b32_e32 v3, v7, v13
+; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
; GISEL-NEXT: v_ffbh_u32_e32 v26, v5
; GISEL-NEXT: v_ffbh_u32_e32 v27, v4
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; GISEL-NEXT: v_ffbh_u32_e32 v28, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v29, v10
+; GISEL-NEXT: v_add_i32_e32 v17, vcc, 32, v17
+; GISEL-NEXT: v_ffbh_u32_e32 v28, v13
+; GISEL-NEXT: v_ffbh_u32_e32 v29, v12
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
-; GISEL-NEXT: v_min_u32_e32 v0, v12, v13
+; GISEL-NEXT: v_min_u32_e32 v0, v14, v15
; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v27
-; GISEL-NEXT: v_min_u32_e32 v2, v14, v15
+; GISEL-NEXT: v_min_u32_e32 v2, v16, v17
; GISEL-NEXT: v_add_i32_e64 v3, s[6:7], 32, v29
; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], 64, v0
; GISEL-NEXT: v_min_u32_e32 v1, v26, v1
@@ -657,32 +657,32 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
; GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v0, v1
; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], 0, 0, s[4:5]
; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[16:17]
+; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[10:11]
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_xor_b32_e32 v12, 0x7f, v2
+; GISEL-NEXT: v_xor_b32_e32 v10, 0x7f, v2
; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v0
-; GISEL-NEXT: v_or_b32_e32 v13, v3, v1
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v0
+; GISEL-NEXT: v_or_b32_e32 v11, v3, v1
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v13, v14, v15
-; GISEL-NEXT: v_and_b32_e32 v14, 1, v13
-; GISEL-NEXT: v_or_b32_e32 v12, v13, v12
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT: v_or_b32_e32 v11, v14, v15
+; GISEL-NEXT: v_and_b32_e32 v14, 1, v11
+; GISEL-NEXT: v_or_b32_e32 v10, v11, v10
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14
; GISEL-NEXT: v_cndmask_b32_e64 v14, v6, 0, vcc
-; GISEL-NEXT: v_and_b32_e32 v16, 1, v12
+; GISEL-NEXT: v_and_b32_e32 v16, 1, v10
; GISEL-NEXT: v_cndmask_b32_e64 v15, v7, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v10, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v13, v11, 0, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v10, v12, 0, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v11, v13, 0, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5]
@@ -695,22 +695,22 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e64 v28, vcc, 0, v0, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v30, v2
-; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], 64, v30
+; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], 64, v30
; GISEL-NEXT: v_lshl_b64 v[0:1], v[6:7], v30
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], v30
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], v30
; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[6:7], v12
+; GISEL-NEXT: v_lshr_b64 v[10:11], v[6:7], v10
; GISEL-NEXT: v_lshl_b64 v[16:17], v[6:7], v14
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
-; GISEL-NEXT: v_or_b32_e32 v0, v12, v2
-; GISEL-NEXT: v_or_b32_e32 v1, v13, v3
+; GISEL-NEXT: v_or_b32_e32 v0, v10, v2
+; GISEL-NEXT: v_or_b32_e32 v1, v11, v3
; GISEL-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v30
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v0, v10, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v1, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v10, v0, v12, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v11, v1, v13, vcc
; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v0, s8
; GISEL-NEXT: v_mov_b32_e32 v1, s9
@@ -722,26 +722,26 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_add_i32_e32 v32, vcc, 0xffffffc0, v26
; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v26
-; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v26
+; GISEL-NEXT: v_lshr_b64 v[0:1], v[12:13], v26
; GISEL-NEXT: v_lshr_b64 v[2:3], v[6:7], v26
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v20
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v21, vcc
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[10:11], v16
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[10:11], v32
+; GISEL-NEXT: v_lshl_b64 v[16:17], v[12:13], v16
+; GISEL-NEXT: v_lshr_b64 v[12:13], v[12:13], v32
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v4, vcc
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v5, vcc
; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
; GISEL-NEXT: v_or_b32_e32 v2, v2, v16
; GISEL-NEXT: v_or_b32_e32 v3, v3, v17
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v26
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v2, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v3, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v12, v2, v6, vcc
+; GISEL-NEXT: v_cndmask_b32_e32 v13, v3, v7, vcc
; GISEL-NEXT: v_mov_b32_e32 v7, 0
; GISEL-NEXT: v_mov_b32_e32 v0, s4
; GISEL-NEXT: v_mov_b32_e32 v1, s5
@@ -749,20 +749,20 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_mov_b32_e32 v3, s7
; GISEL-NEXT: .LBB0_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
+; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], 1
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v11
-; GISEL-NEXT: v_lshrrev_b32_e32 v34, 31, v13
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[14:15], 1
-; GISEL-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v13
+; GISEL-NEXT: v_lshrrev_b32_e32 v34, 31, v11
+; GISEL-NEXT: v_lshl_b64 v[12:13], v[14:15], 1
+; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v14, 31, v15
; GISEL-NEXT: v_add_i32_e32 v26, vcc, -1, v26
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, -1, v27, vcc
; GISEL-NEXT: v_or_b32_e32 v16, v16, v6
; GISEL-NEXT: v_or_b32_e32 v2, v2, v34
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v14
-; GISEL-NEXT: v_or_b32_e32 v14, v0, v10
-; GISEL-NEXT: v_or_b32_e32 v15, v1, v11
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v14
+; GISEL-NEXT: v_or_b32_e32 v14, v0, v12
+; GISEL-NEXT: v_or_b32_e32 v15, v1, v13
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, -1, v28, vcc
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v30, v2
@@ -775,14 +775,14 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v6
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_and_b32_e32 v6, 1, v0
-; GISEL-NEXT: v_and_b32_e32 v10, v0, v20
-; GISEL-NEXT: v_and_b32_e32 v11, v0, v21
+; GISEL-NEXT: v_and_b32_e32 v12, v0, v20
+; GISEL-NEXT: v_and_b32_e32 v13, v0, v21
; GISEL-NEXT: v_and_b32_e32 v34, v0, v4
; GISEL-NEXT: v_and_b32_e32 v35, v0, v5
; GISEL-NEXT: v_mov_b32_e32 v0, v6
; GISEL-NEXT: v_mov_b32_e32 v1, v7
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v2, v10
-; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v11, vcc
+; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v2, v12
+; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v3, v13, vcc
; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v16, v34, vcc
; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v35, vcc
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -792,9 +792,9 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: .LBB0_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
; GISEL-NEXT: v_lshl_b64 v[2:3], v[14:15], 1
-; GISEL-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
+; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v15
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v4
+; GISEL-NEXT: v_or_b32_e32 v10, v10, v4
; GISEL-NEXT: v_or_b32_e32 v14, v0, v2
; GISEL-NEXT: v_or_b32_e32 v15, v1, v3
; GISEL-NEXT: .LBB0_12: ; %Flow12
@@ -807,8 +807,8 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_xor_b32_e32 v6, v9, v3
; GISEL-NEXT: v_xor_b32_e32 v4, v14, v7
; GISEL-NEXT: v_xor_b32_e32 v5, v15, v7
-; GISEL-NEXT: v_xor_b32_e32 v8, v12, v7
-; GISEL-NEXT: v_xor_b32_e32 v9, v13, v7
+; GISEL-NEXT: v_xor_b32_e32 v8, v10, v7
+; GISEL-NEXT: v_xor_b32_e32 v9, v11, v7
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v4, v7
diff --git a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
index af9dbc5052dc2..1beed46fcfabd 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll
@@ -61,12 +61,13 @@ define amdgpu_kernel void @write_ds_sub0_offset0_global_clamp_bit(float %dummy.v
; CI-NEXT: s_mov_b64 vcc, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0
-; CI-NEXT: v_mov_b32_e32 v2, 0x7b
+; CI-NEXT: s_mov_b32 s0, 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1
+; CI-NEXT: v_mov_b32_e32 v2, 0x7b
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b64 s[0:1], 0
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: ds_write_b32 v0, v2 offset:12
; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -142,9 +143,10 @@ define amdgpu_kernel void @write_ds_sub_max_offset_global_clamp_bit(float %dummy
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_div_fmas_f32 v0, v0, v0, v0
-; CI-NEXT: s_mov_b64 s[0:1], 0
+; CI-NEXT: s_mov_b32 s0, 0
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: ds_write_b32 v2, v1
; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -564,13 +566,14 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_clamp_
; CI-NEXT: s_mov_b64 vcc, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s0
-; CI-NEXT: v_mov_b32_e32 v2, 0x7b
+; CI-NEXT: s_mov_b32 s0, 0
; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1
+; CI-NEXT: v_mov_b32_e32 v2, 0x7b
; CI-NEXT: v_mov_b32_e32 v3, 0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b64 s[0:1], 0
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s1, s0
; CI-NEXT: ds_write2_b32 v0, v2, v3 offset1:1
; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0
; CI-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fceil64.ll b/llvm/test/CodeGen/AMDGPU/fceil64.ll
index bd1f98a39c252..35b07fe7e7a6f 100644
--- a/llvm/test/CodeGen/AMDGPU/fceil64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fceil64.ll
@@ -230,12 +230,12 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_bfe_u32 s33, s3, 0xb0014
; SI-NEXT: s_and_b32 s40, s3, 0x80000000
; SI-NEXT: v_cmp_gt_f64_e64 s[22:23], s[2:3], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[26:27], s[0:1], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[30:31], s[6:7], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[36:37], s[4:5], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[24:25], s[10:11], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[28:29], s[8:9], 0
-; SI-NEXT: v_cmp_gt_f64_e64 s[34:35], s[14:15], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[24:25], s[0:1], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[28:29], s[6:7], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[34:35], s[4:5], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[26:27], s[10:11], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[30:31], s[8:9], 0
+; SI-NEXT: v_cmp_gt_f64_e64 s[36:37], s[14:15], 0
; SI-NEXT: s_addk_i32 s33, 0xfc01
; SI-NEXT: s_lshr_b64 s[38:39], s[20:21], s33
; SI-NEXT: s_andn2_b64 s[38:39], s[2:3], s[38:39]
@@ -268,7 +268,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[0:1], v[0:1]
-; SI-NEXT: s_and_b64 s[0:1], s[26:27], vcc
+; SI-NEXT: s_and_b64 s[0:1], s[24:25], vcc
; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; SI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s1, s7, 0xb0014
@@ -287,7 +287,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v6, s1
; SI-NEXT: v_mov_b32_e32 v5, s0
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[6:7], v[5:6]
-; SI-NEXT: s_and_b64 s[2:3], s[30:31], vcc
+; SI-NEXT: s_and_b64 s[2:3], s[28:29], vcc
; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; SI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s3, s5, 0xb0014
@@ -306,7 +306,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v6, s3
; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[4:5], v[5:6]
-; SI-NEXT: s_and_b64 s[0:1], s[36:37], vcc
+; SI-NEXT: s_and_b64 s[0:1], s[34:35], vcc
; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; SI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s1, s11, 0xb0014
@@ -327,7 +327,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v6, s1
; SI-NEXT: v_mov_b32_e32 v5, s0
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[10:11], v[5:6]
-; SI-NEXT: s_and_b64 s[2:3], s[24:25], vcc
+; SI-NEXT: s_and_b64 s[2:3], s[26:27], vcc
; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; SI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s3, s9, 0xb0014
@@ -346,7 +346,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v6, s3
; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[8:9], v[5:6]
-; SI-NEXT: s_and_b64 s[0:1], s[28:29], vcc
+; SI-NEXT: s_and_b64 s[0:1], s[30:31], vcc
; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; SI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s1, s15, 0xb0014
@@ -365,7 +365,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: v_mov_b32_e32 v10, s1
; SI-NEXT: v_mov_b32_e32 v9, s0
; SI-NEXT: v_cmp_lg_f64_e32 vcc, s[14:15], v[9:10]
-; SI-NEXT: s_and_b64 s[2:3], s[34:35], vcc
+; SI-NEXT: s_and_b64 s[2:3], s[36:37], vcc
; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; SI-NEXT: s_cselect_b32 s4, 0x3ff00000, 0
; SI-NEXT: s_bfe_u32 s2, s13, 0xb0014
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index f01879d98da41..4f9356300df95 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -5980,52 +5980,48 @@ define double @flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_3
-; GFX908-NEXT: ; %bb.1: ; %Flow3
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_6
-; GFX908-NEXT: .LBB30_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB30_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB30_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB30_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB30_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB30_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB30_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB30_4: ; %Flow3
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB30_2
-; GFX908-NEXT: .LBB30_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB30_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_add_f64 v[2:3], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_add_f64 v[0:1], v[4:5], v[2:3]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB30_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 1924751427da0..546a8bb4c45b2 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -3480,52 +3480,48 @@ define double @flat_agent_atomic_fsub_ret_f64(ptr %ptr, double %val) #0 {
; GFX908-LABEL: flat_agent_atomic_fsub_ret_f64:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_3
-; GFX908-NEXT: ; %bb.1: ; %Flow3
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_6
-; GFX908-NEXT: .LBB16_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB16_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB16_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB16_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], -v[2:3]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_add_f64 v[4:5], v[6:7], -v[2:3]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB16_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB16_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB16_4: ; %Flow3
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB16_2
-; GFX908-NEXT: .LBB16_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB16_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_add_f64 v[2:3], v[0:1], -v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_add_f64 v[0:1], v[4:5], -v[2:3]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB16_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
index e5187a811a230..492d2bfd0e44d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
@@ -864,28 +864,26 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v3, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v7, v1
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
-; GFX7-NEXT: v_and_b32_e32 v5, s5, v7
-; GFX7-NEXT: v_and_b32_e32 v4, s4, v6
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_and_i64_ret:
@@ -896,28 +894,26 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_and_b32_e32 v5, s5, v7
-; GFX8-NEXT: v_and_b32_e32 v4, s4, v6
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret:
@@ -4944,28 +4940,26 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v3, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v7, v1
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
-; GFX7-NEXT: v_or_b32_e32 v5, s5, v7
-; GFX7-NEXT: v_or_b32_e32 v4, s4, v6
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_or_i64_ret:
@@ -4976,28 +4970,26 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_or_b32_e32 v5, s5, v7
-; GFX8-NEXT: v_or_b32_e32 v4, s4, v6
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret:
@@ -6142,28 +6134,26 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v3, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX7-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v7, v1
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
-; GFX7-NEXT: v_xor_b32_e32 v5, s5, v7
-; GFX7-NEXT: v_xor_b32_e32 v4, s4, v6
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB79_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_xor_i64_ret:
@@ -6174,28 +6164,26 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, v0
-; GFX8-NEXT: v_xor_b32_e32 v5, s5, v7
-; GFX8-NEXT: v_xor_b32_e32 v4, s4, v6
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB79_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret:
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
index 9e27f6badfdac..4ad13c01ca07d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
@@ -12786,56 +12786,52 @@ define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_max_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB82_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB82_6
-; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB82_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
-; GCN3-NEXT: .LBB82_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB82_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT: s_cbranch_execnz .LBB82_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB82_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB82_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execz .LBB82_2
-; GCN3-NEXT: .LBB82_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB82_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
+; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB82_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw max ptr %ptr, i64 %in seq_cst
@@ -15385,56 +15381,52 @@ define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_umax_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB96_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB96_6
-; GCN3-NEXT: .LBB96_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB96_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB96_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
-; GCN3-NEXT: .LBB96_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB96_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT: s_cbranch_execnz .LBB96_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB96_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB96_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execz .LBB96_2
-; GCN3-NEXT: .LBB96_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB96_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
+; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB96_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst
@@ -17775,56 +17767,52 @@ define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_umin_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB109_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB109_6
-; GCN3-NEXT: .LBB109_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB109_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB109_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
-; GCN3-NEXT: .LBB109_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB109_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT: s_cbranch_execnz .LBB109_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB109_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB109_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execz .LBB109_2
-; GCN3-NEXT: .LBB109_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB109_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
+; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB109_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst
@@ -19522,56 +19510,52 @@ define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_min_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB119_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB119_6
-; GCN3-NEXT: .LBB119_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB119_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB119_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
-; GCN3-NEXT: .LBB119_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB119_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT: s_cbranch_execnz .LBB119_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB119_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB119_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execz .LBB119_2
-; GCN3-NEXT: .LBB119_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB119_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc
+; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB119_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw min ptr %ptr, i64 %in seq_cst
@@ -22148,61 +22132,57 @@ define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB133_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB133_6
-; GCN3-NEXT: .LBB133_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB133_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB133_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
-; GCN3-NEXT: .LBB133_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB133_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v8
-; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 1, v6
+; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT: s_cbranch_execnz .LBB133_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB133_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB133_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT: s_cbranch_execz .LBB133_2
-; GCN3-NEXT: .LBB133_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB133_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
-; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
+; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, 1, v4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
+; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v5, vcc
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3]
+; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB133_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
@@ -24011,65 +23991,61 @@ define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
-; GCN3-NEXT: v_mov_b32_e32 v4, v0
-; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GCN3-NEXT: s_cbranch_execnz .LBB143_3
-; GCN3-NEXT: ; %bb.1: ; %Flow3
-; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
-; GCN3-NEXT: s_cbranch_execnz .LBB143_6
-; GCN3-NEXT: .LBB143_2: ; %atomicrmw.phi
-; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN3-NEXT: s_setpc_b64 s[30:31]
-; GCN3-NEXT: .LBB143_3: ; %atomicrmw.global
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT: s_cbranch_execz .LBB143_4
+; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[10:11], 0
-; GCN3-NEXT: .LBB143_4: ; %atomicrmw.start
+; GCN3-NEXT: .LBB143_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v9, v1
-; GCN3-NEXT: v_mov_b32_e32 v8, v0
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
-; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v8
-; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GCN3-NEXT: v_mov_b32_e32 v7, v5
+; GCN3-NEXT: v_mov_b32_e32 v6, v4
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN3-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GCN3-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11]
-; GCN3-NEXT: s_cbranch_execnz .LBB143_4
-; GCN3-NEXT: ; %bb.5: ; %Flow
+; GCN3-NEXT: s_cbranch_execnz .LBB143_2
+; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[10:11]
-; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GCN3-NEXT: .LBB143_4: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
-; GCN3-NEXT: s_cbranch_execz .LBB143_2
-; GCN3-NEXT: .LBB143_6: ; %atomicrmw.private
-; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: s_cbranch_execz .LBB143_6
+; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
-; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0
+; GCN3-NEXT: v_add_co_u32_e64 v1, s[6:7], -1, v4
; GCN3-NEXT: s_waitcnt vmcnt(0)
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
-; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3]
+; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v5, s[6:7]
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
-; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
-; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
+; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
+; GCN3-NEXT: .LBB143_6: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
+; GCN3-NEXT: v_mov_b32_e32 v0, v4
+; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 4233430eafb83..f30c3f78d7f5b 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -500,9 +500,9 @@ define i128 @fptosi_f32_to_i128(float %x) {
; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GISEL-NEXT: v_lshl_or_b32 v9, v0, 16, v0
-; GISEL-NEXT: v_or3_b32 v7, v1, v2, 1
-; GISEL-NEXT: v_or3_b32 v8, v0, v2, 0
+; GISEL-NEXT: v_lshl_or_b32 v8, v0, 16, v0
+; GISEL-NEXT: v_or3_b32 v9, v1, v2, 1
+; GISEL-NEXT: v_or3_b32 v7, v0, v2, 0
; GISEL-NEXT: v_mov_b32_e32 v0, 0x7fffff
; GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
; GISEL-NEXT: v_and_or_b32 v4, v4, v0, v1
@@ -520,30 +520,30 @@ define i128 @fptosi_f32_to_i128(float %x) {
; GISEL-NEXT: v_sub_u32_e32 v2, 64, v14
; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v0, vcc
; GISEL-NEXT: v_lshrrev_b64 v[10:11], v2, v[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v15, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v15, v8, 0
; GISEL-NEXT: v_add_u32_e32 v6, 0xffffff2a, v6
; GISEL-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v6, v8, v[2:3]
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v6, v7, v[2:3]
; GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v10, vcc
; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
; GISEL-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5]
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v15, v7, 0
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v4, v7, v[12:13]
-; GISEL-NEXT: v_mul_lo_u32 v10, v6, v9
-; GISEL-NEXT: v_mul_lo_u32 v9, v15, v9
-; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v15, v8, v[1:2]
-; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v6, v7, v[12:13]
-; GISEL-NEXT: v_addc_co_u32_e64 v3, s[8:9], v3, v9, s[8:9]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v15, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v4, v9, v[12:13]
+; GISEL-NEXT: v_mul_lo_u32 v10, v6, v8
+; GISEL-NEXT: v_mul_lo_u32 v8, v15, v8
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v15, v7, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v6, v9, v[12:13]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[8:9], v3, v8, s[8:9]
; GISEL-NEXT: v_addc_co_u32_e64 v3, s[6:7], v3, v10, s[6:7]
-; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v4, v8, v[3:4]
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v4, v7, v[3:4]
; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v11, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
; GISEL-NEXT: ; implicit-def: $vgpr6
; GISEL-NEXT: ; implicit-def: $vgpr8
-; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v7, v[9:10]
-; GISEL-NEXT: ; implicit-def: $vgpr4
; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v9, v[12:13]
+; GISEL-NEXT: ; implicit-def: $vgpr4
; GISEL-NEXT: ; implicit-def: $vgpr9
; GISEL-NEXT: .LBB2_3: ; %Flow
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[12:13]
@@ -551,11 +551,11 @@ define i128 @fptosi_f32_to_i128(float %x) {
; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
; GISEL-NEXT: v_sub_u32_e32 v0, 0x96, v6
; GISEL-NEXT: v_lshrrev_b32_e32 v6, v0, v4
-; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v7, 0
-; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v6, v9, 0
-; GISEL-NEXT: v_mul_lo_u32 v7, v6, v9
-; GISEL-NEXT: v_mad_u64_u32 v[4:5], vcc, v6, v8, v[1:2]
-; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v9, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v6, v8, 0
+; GISEL-NEXT: v_mul_lo_u32 v8, v6, v8
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], vcc, v6, v7, v[1:2]
+; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: .LBB2_5: ; %Flow1
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index afc6d644beda2..48c3538d98ab9 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -544,62 +544,62 @@ entry:
define amdgpu_kernel void @fptrunc_f64_to_f16(
; SI-SDAG-LABEL: fptrunc_f64_to_f16:
; SI-SDAG: ; %bb.0: ; %entry
-; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: s_mov_b32 s10, s2
-; SI-SDAG-NEXT: s_mov_b32 s11, s3
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s10, s6
+; SI-SDAG-NEXT: s_mov_b32 s11, s7
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b32 s8, s6
-; SI-SDAG-NEXT: s_mov_b32 s9, s7
+; SI-SDAG-NEXT: s_mov_b32 s8, s2
+; SI-SDAG-NEXT: s_mov_b32 s9, s3
; SI-SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; SI-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; SI-SDAG-NEXT: s_movk_i32 s2, 0x7e00
; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; SI-SDAG-NEXT: v_readfirstlane_b32 s1, v1
-; SI-SDAG-NEXT: s_and_b32 s6, s1, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s7, s1, 8
-; SI-SDAG-NEXT: s_bfe_u32 s8, s1, 0xb0014
-; SI-SDAG-NEXT: v_or_b32_e32 v0, s6, v0
-; SI-SDAG-NEXT: s_and_b32 s6, s7, 0xffe
-; SI-SDAG-NEXT: s_sub_i32 s7, 0x3f1, s8
+; SI-SDAG-NEXT: v_readfirstlane_b32 s3, v1
+; SI-SDAG-NEXT: s_and_b32 s4, s3, 0x1ff
+; SI-SDAG-NEXT: s_lshr_b32 s5, s3, 8
+; SI-SDAG-NEXT: s_bfe_u32 s8, s3, 0xb0014
+; SI-SDAG-NEXT: v_or_b32_e32 v0, s4, v0
+; SI-SDAG-NEXT: s_and_b32 s4, s5, 0xffe
+; SI-SDAG-NEXT: s_sub_i32 s5, 0x3f1, s8
; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-SDAG-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
+; SI-SDAG-NEXT: v_med3_i32 v1, s5, 0, 13
+; SI-SDAG-NEXT: v_readfirstlane_b32 s5, v0
; SI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; SI-SDAG-NEXT: s_or_b32 s6, s6, s7
-; SI-SDAG-NEXT: s_or_b32 s7, s6, 0x1000
-; SI-SDAG-NEXT: s_lshr_b32 s10, s7, s9
+; SI-SDAG-NEXT: s_or_b32 s4, s4, s5
+; SI-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-SDAG-NEXT: s_lshr_b32 s10, s5, s9
; SI-SDAG-NEXT: s_lshl_b32 s9, s10, s9
-; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s7
-; SI-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s5
+; SI-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; SI-SDAG-NEXT: s_addk_i32 s8, 0xfc10
-; SI-SDAG-NEXT: s_or_b32 s7, s10, s7
+; SI-SDAG-NEXT: s_or_b32 s5, s10, s5
; SI-SDAG-NEXT: s_lshl_b32 s9, s8, 12
-; SI-SDAG-NEXT: s_or_b32 s9, s6, s9
+; SI-SDAG-NEXT: s_or_b32 s9, s4, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s8, 1
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, s9
-; SI-SDAG-NEXT: s_and_b32 s9, s7, 7
+; SI-SDAG-NEXT: s_cselect_b32 s5, s5, s9
+; SI-SDAG-NEXT: s_and_b32 s9, s5, 7
; SI-SDAG-NEXT: s_cmp_gt_i32 s9, 5
; SI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
; SI-SDAG-NEXT: s_cmp_eq_u32 s9, 3
; SI-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; SI-SDAG-NEXT: s_lshr_b32 s7, s7, 2
+; SI-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; SI-SDAG-NEXT: s_or_b32 s9, s9, s10
-; SI-SDAG-NEXT: s_add_i32 s7, s7, s9
+; SI-SDAG-NEXT: s_add_i32 s5, s5, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s8, 31
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s6, 0
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, 0x7c00
+; SI-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; SI-SDAG-NEXT: s_cselect_b32 s2, s2, 0x7c00
; SI-SDAG-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s7
-; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 16
-; SI-SDAG-NEXT: s_and_b32 s1, s1, 0x8000
-; SI-SDAG-NEXT: s_or_b32 s6, s1, s0
-; SI-SDAG-NEXT: s_mov_b32 s0, s4
-; SI-SDAG-NEXT: s_mov_b32 s1, s5
-; SI-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; SI-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_cselect_b32 s2, s2, s5
+; SI-SDAG-NEXT: s_lshr_b32 s3, s3, 16
+; SI-SDAG-NEXT: s_and_b32 s3, s3, 0x8000
+; SI-SDAG-NEXT: s_or_b32 s2, s3, s2
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; SI-SDAG-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: fptrunc_f64_to_f16:
@@ -2051,32 +2051,32 @@ entry:
define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-LABEL: fptrunc_v2f64_to_v2f16:
; SI-SDAG: ; %bb.0: ; %entry
-; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: s_mov_b32 s10, s2
-; SI-SDAG-NEXT: s_mov_b32 s11, s3
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
+; SI-SDAG-NEXT: s_mov_b32 s10, s6
+; SI-SDAG-NEXT: s_mov_b32 s11, s7
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b32 s8, s6
-; SI-SDAG-NEXT: s_mov_b32 s9, s7
+; SI-SDAG-NEXT: s_mov_b32 s8, s2
+; SI-SDAG-NEXT: s_mov_b32 s9, s3
; SI-SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; SI-SDAG-NEXT: s_movk_i32 s2, 0x7e00
; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; SI-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; SI-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; SI-SDAG-NEXT: s_and_b32 s7, s1, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s8, s1, 8
-; SI-SDAG-NEXT: s_bfe_u32 s9, s1, 0xb0014
-; SI-SDAG-NEXT: v_or_b32_e32 v1, s7, v2
-; SI-SDAG-NEXT: s_and_b32 s7, s8, 0xffe
+; SI-SDAG-NEXT: v_readfirstlane_b32 s3, v3
+; SI-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; SI-SDAG-NEXT: s_and_b32 s5, s3, 0x1ff
+; SI-SDAG-NEXT: s_lshr_b32 s8, s3, 8
+; SI-SDAG-NEXT: s_bfe_u32 s9, s3, 0xb0014
+; SI-SDAG-NEXT: v_or_b32_e32 v1, s5, v2
+; SI-SDAG-NEXT: s_and_b32 s5, s8, 0xffe
; SI-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s9
; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; SI-SDAG-NEXT: v_med3_i32 v2, s8, 0, 13
; SI-SDAG-NEXT: v_readfirstlane_b32 s8, v1
; SI-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; SI-SDAG-NEXT: s_or_b32 s7, s7, s8
-; SI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
+; SI-SDAG-NEXT: s_or_b32 s5, s5, s8
+; SI-SDAG-NEXT: s_or_b32 s8, s5, 0x1000
; SI-SDAG-NEXT: s_lshr_b32 s11, s8, s10
; SI-SDAG-NEXT: s_lshl_b32 s10, s11, s10
; SI-SDAG-NEXT: s_cmp_lg_u32 s10, s8
@@ -2084,7 +2084,7 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-NEXT: s_addk_i32 s9, 0xfc10
; SI-SDAG-NEXT: s_or_b32 s8, s11, s8
; SI-SDAG-NEXT: s_lshl_b32 s10, s9, 12
-; SI-SDAG-NEXT: s_or_b32 s10, s7, s10
+; SI-SDAG-NEXT: s_or_b32 s10, s5, s10
; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 1
; SI-SDAG-NEXT: s_cselect_b32 s8, s8, s10
; SI-SDAG-NEXT: s_and_b32 s10, s8, 7
@@ -2097,27 +2097,27 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-NEXT: s_add_i32 s8, s8, s10
; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 31
; SI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; SI-SDAG-NEXT: s_cselect_b32 s7, s0, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s5, 0
+; SI-SDAG-NEXT: s_cselect_b32 s5, s2, 0x7c00
; SI-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, s8
-; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 16
-; SI-SDAG-NEXT: s_and_b32 s8, s6, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s9, s6, 8
-; SI-SDAG-NEXT: s_bfe_u32 s10, s6, 0xb0014
-; SI-SDAG-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-SDAG-NEXT: s_cselect_b32 s5, s5, s8
+; SI-SDAG-NEXT: s_lshr_b32 s3, s3, 16
+; SI-SDAG-NEXT: s_and_b32 s8, s4, 0x1ff
+; SI-SDAG-NEXT: s_lshr_b32 s9, s4, 8
+; SI-SDAG-NEXT: s_bfe_u32 s10, s4, 0xb0014
+; SI-SDAG-NEXT: s_and_b32 s3, s3, 0x8000
; SI-SDAG-NEXT: v_or_b32_e32 v0, s8, v0
; SI-SDAG-NEXT: s_and_b32 s8, s9, 0xffe
; SI-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s10
-; SI-SDAG-NEXT: s_or_b32 s1, s1, s7
+; SI-SDAG-NEXT: s_or_b32 s3, s3, s5
; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SI-SDAG-NEXT: v_med3_i32 v1, s9, 0, 13
-; SI-SDAG-NEXT: s_lshl_b32 s1, s1, 16
-; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
+; SI-SDAG-NEXT: s_lshl_b32 s3, s3, 16
+; SI-SDAG-NEXT: v_readfirstlane_b32 s5, v0
; SI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; SI-SDAG-NEXT: s_or_b32 s7, s8, s7
-; SI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
+; SI-SDAG-NEXT: s_or_b32 s5, s8, s5
+; SI-SDAG-NEXT: s_or_b32 s8, s5, 0x1000
; SI-SDAG-NEXT: s_lshr_b32 s11, s8, s9
; SI-SDAG-NEXT: s_lshl_b32 s9, s11, s9
; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s8
@@ -2125,7 +2125,7 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-NEXT: s_addk_i32 s10, 0xfc10
; SI-SDAG-NEXT: s_or_b32 s8, s11, s8
; SI-SDAG-NEXT: s_lshl_b32 s9, s10, 12
-; SI-SDAG-NEXT: s_or_b32 s9, s7, s9
+; SI-SDAG-NEXT: s_or_b32 s9, s5, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s10, 1
; SI-SDAG-NEXT: s_cselect_b32 s8, s8, s9
; SI-SDAG-NEXT: s_and_b32 s9, s8, 7
@@ -2138,19 +2138,19 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-NEXT: s_add_i32 s8, s8, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s10, 31
; SI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s5, 0
+; SI-SDAG-NEXT: s_cselect_b32 s2, s2, 0x7c00
; SI-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s8
-; SI-SDAG-NEXT: s_lshr_b32 s6, s6, 16
-; SI-SDAG-NEXT: s_and_b32 s6, s6, 0x8000
-; SI-SDAG-NEXT: s_or_b32 s0, s6, s0
-; SI-SDAG-NEXT: s_and_b32 s0, s0, 0xffff
-; SI-SDAG-NEXT: s_or_b32 s6, s0, s1
-; SI-SDAG-NEXT: s_mov_b32 s0, s4
-; SI-SDAG-NEXT: s_mov_b32 s1, s5
-; SI-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_cselect_b32 s2, s2, s8
+; SI-SDAG-NEXT: s_lshr_b32 s4, s4, 16
+; SI-SDAG-NEXT: s_and_b32 s4, s4, 0x8000
+; SI-SDAG-NEXT: s_or_b32 s2, s4, s2
+; SI-SDAG-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-SDAG-NEXT: s_or_b32 s2, s2, s3
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: fptrunc_v2f64_to_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index 3e344af903a1e..332ce677b0882 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -174,55 +174,55 @@ define amdgpu_kernel void @fptrunc_f64_to_f32_afn(ptr addrspace(1) %out, double
define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in) {
; SI-LABEL: fptrunc_f64_to_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_movk_i32 s2, 0x7e00
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_movk_i32 s6, 0x7e00
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s0, s7, 8
-; SI-NEXT: s_and_b32 s1, s7, 0x1ff
-; SI-NEXT: s_and_b32 s8, s0, 0xffe
-; SI-NEXT: s_or_b32 s0, s1, s6
-; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: s_bfe_u32 s0, s7, 0xb0014
-; SI-NEXT: v_readfirstlane_b32 s1, v0
-; SI-NEXT: s_sub_i32 s6, 0x3f1, s0
-; SI-NEXT: s_or_b32 s1, s8, s1
-; SI-NEXT: v_med3_i32 v0, s6, 0, 13
-; SI-NEXT: s_or_b32 s6, s1, 0x1000
+; SI-NEXT: s_lshr_b32 s4, s3, 8
+; SI-NEXT: s_and_b32 s5, s3, 0x1ff
+; SI-NEXT: s_and_b32 s8, s4, 0xffe
+; SI-NEXT: s_or_b32 s2, s5, s2
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT: s_bfe_u32 s2, s3, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s4, v0
+; SI-NEXT: s_sub_i32 s5, 0x3f1, s2
+; SI-NEXT: s_or_b32 s4, s8, s4
+; SI-NEXT: v_med3_i32 v0, s5, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_lshr_b32 s9, s6, s8
+; SI-NEXT: s_lshr_b32 s9, s5, s8
; SI-NEXT: s_lshl_b32 s8, s9, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s6
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_addk_i32 s0, 0xfc10
-; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_lshl_b32 s8, s0, 12
-; SI-NEXT: s_or_b32 s8, s1, s8
-; SI-NEXT: s_cmp_lt_i32 s0, 1
-; SI-NEXT: s_cselect_b32 s6, s6, s8
-; SI-NEXT: s_and_b32 s8, s6, 7
+; SI-NEXT: s_cmp_lg_u32 s8, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s2, 0xfc10
+; SI-NEXT: s_or_b32 s5, s9, s5
+; SI-NEXT: s_lshl_b32 s8, s2, 12
+; SI-NEXT: s_or_b32 s8, s4, s8
+; SI-NEXT: s_cmp_lt_i32 s2, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s8
+; SI-NEXT: s_and_b32 s8, s5, 7
; SI-NEXT: s_cmp_gt_i32 s8, 5
; SI-NEXT: s_cselect_b32 s9, 1, 0
; SI-NEXT: s_cmp_eq_u32 s8, 3
; SI-NEXT: s_cselect_b32 s8, 1, 0
-; SI-NEXT: s_lshr_b32 s6, s6, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_add_i32 s6, s6, s8
-; SI-NEXT: s_cmp_lt_i32 s0, 31
-; SI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s1, 0
-; SI-NEXT: s_cselect_b32 s1, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s0, 0x40f
-; SI-NEXT: s_cselect_b32 s0, s1, s6
-; SI-NEXT: s_lshr_b32 s1, s7, 16
-; SI-NEXT: s_and_b32 s1, s1, 0x8000
-; SI-NEXT: s_or_b32 s6, s1, s0
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s0, s4
-; SI-NEXT: s_mov_b32 s1, s5
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_add_i32 s5, s5, s8
+; SI-NEXT: s_cmp_lt_i32 s2, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s2, 0x40f
+; SI-NEXT: s_cselect_b32 s2, s4, s5
+; SI-NEXT: s_lshr_b32 s3, s3, 16
+; SI-NEXT: s_and_b32 s3, s3, 0x8000
+; SI-NEXT: s_or_b32 s2, s3, s2
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_f64_to_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 3928a44595761..14477bd8f0505 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -914,10 +914,11 @@ define void @void_func_v2i8(<2 x i8> %arg0) #0 {
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; CI-NEXT: v_and_b32_e32 v0, 0xff, v0
+; CI-NEXT: s_mov_b32 s4, 0
; CI-NEXT: v_or_b32_e32 v0, v0, v1
-; CI-NEXT: s_mov_b64 s[4:5], 0
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_mov_b32 s5, s4
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -926,10 +927,11 @@ define void @void_func_v2i8(<2 x i8> %arg0) #0 {
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX89-NEXT: s_mov_b32 s4, 0
; GFX89-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX89-NEXT: s_mov_b64 s[4:5], 0
; GFX89-NEXT: s_mov_b32 s7, 0xf000
; GFX89-NEXT: s_mov_b32 s6, -1
+; GFX89-NEXT: s_mov_b32 s5, s4
; GFX89-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX89-NEXT: s_waitcnt vmcnt(0)
; GFX89-NEXT: s_setpc_b64 s[30:31]
@@ -939,10 +941,10 @@ define void @void_func_v2i8(<2 x i8> %arg0) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s0
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -952,10 +954,10 @@ define void @void_func_v2i8(<2 x i8> %arg0) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
@@ -990,12 +992,13 @@ define void @void_func_v3i8(<3 x i8> %arg0) #0 {
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; CI-NEXT: v_and_b32_e32 v0, 0xff, v0
-; CI-NEXT: s_mov_b64 s[4:5], 2
+; CI-NEXT: s_mov_b32 s5, 0
+; CI-NEXT: s_mov_b32 s4, 2
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: buffer_store_byte v2, off, s[4:7], 0
-; CI-NEXT: s_mov_b64 s[4:5], 0
+; CI-NEXT: s_mov_b32 s4, s5
; CI-NEXT: buffer_store_short v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1005,11 +1008,12 @@ define void @void_func_v3i8(<3 x i8> %arg0) #0 {
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX89-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX89-NEXT: s_mov_b64 s[4:5], 2
+; GFX89-NEXT: s_mov_b32 s5, 0
+; GFX89-NEXT: s_mov_b32 s4, 2
; GFX89-NEXT: s_mov_b32 s7, 0xf000
; GFX89-NEXT: s_mov_b32 s6, -1
; GFX89-NEXT: buffer_store_byte v2, off, s[4:7], 0
-; GFX89-NEXT: s_mov_b64 s[4:5], 0
+; GFX89-NEXT: s_mov_b32 s4, s5
; GFX89-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX89-NEXT: s_waitcnt vmcnt(0)
; GFX89-NEXT: s_setpc_b64 s[30:31]
@@ -1018,11 +1022,12 @@ define void @void_func_v3i8(<3 x i8> %arg0) #0 {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-NEXT: s_mov_b64 s[0:1], 2
+; GFX11-NEXT: s_mov_b32 s1, 0
+; GFX11-NEXT: s_mov_b32 s0, 2
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: buffer_store_b8 v2, off, s[0:3], 0
-; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_mov_b32 s0, s1
; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-NEXT: s_setpc_b64 s[30:31]
store <3 x i8> %arg0, ptr addrspace(1) null
@@ -1041,10 +1046,11 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 {
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; CI-NEXT: s_mov_b32 s4, 0
; CI-NEXT: v_or_b32_e32 v0, v0, v2
-; CI-NEXT: s_mov_b64 s[4:5], 0
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_mov_b32 s5, s4
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1056,10 +1062,11 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 {
; VI-NEXT: v_perm_b32 v0, v0, v1, s4
; VI-NEXT: v_perm_b32 v1, v2, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: s_mov_b32 s4, 0
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_mov_b32 s5, s4
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1070,10 +1077,11 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 {
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_perm_b32 v1, v2, v3, s4
+; GFX9-NEXT: s_mov_b32 s4, 0
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s4
; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1083,10 +1091,10 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_mov_b32 s1, s0
; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -1106,12 +1114,13 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 {
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; CI-NEXT: s_mov_b64 s[4:5], 4
+; CI-NEXT: s_mov_b32 s5, 0
+; CI-NEXT: s_mov_b32 s4, 4
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: v_or_b32_e32 v0, v0, v2
; CI-NEXT: buffer_store_byte v4, off, s[4:7], 0
-; CI-NEXT: s_mov_b64 s[4:5], 0
+; CI-NEXT: s_mov_b32 s4, s5
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1123,12 +1132,13 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 {
; VI-NEXT: v_perm_b32 v0, v0, v1, s4
; VI-NEXT: v_perm_b32 v1, v2, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: s_mov_b64 s[4:5], 4
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_mov_b32 s4, 4
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: v_or_b32_e32 v0, v0, v1
; VI-NEXT: buffer_store_byte v4, off, s[4:7], 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_mov_b32 s4, s5
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1139,12 +1149,13 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 {
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_perm_b32 v1, v2, v3, s4
-; GFX9-NEXT: s_mov_b64 s[4:5], 4
+; GFX9-NEXT: s_mov_b32 s5, 0
+; GFX9-NEXT: s_mov_b32 s4, 4
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_byte v4, off, s[4:7], 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_mov_b32 s4, s5
; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1154,13 +1165,13 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-NEXT: s_mov_b64 s[0:1], 4
+; GFX11-NEXT: s_mov_b32 s1, 0
+; GFX11-NEXT: s_mov_b32 s0, 4
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-NEXT: buffer_store_b8 v4, off, s[0:3], 0
-; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_mov_b32 s0, s1
; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX11-NEXT: s_setpc_b64 s[30:31]
store <5 x i8> %arg0, ptr addrspace(1) null
@@ -1187,11 +1198,12 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 {
; CI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; CI-NEXT: s_mov_b32 s4, 0
; CI-NEXT: v_or_b32_e32 v4, v4, v6
; CI-NEXT: v_or_b32_e32 v3, v0, v2
-; CI-NEXT: s_mov_b64 s[4:5], 0
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_mov_b32 s5, s4
; CI-NEXT: buffer_store_dwordx2 v[3:4], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1206,11 +1218,12 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 {
; VI-NEXT: v_perm_b32 v1, v2, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: s_mov_b32 s4, 0
; VI-NEXT: v_or_b32_e32 v4, v4, v5
; VI-NEXT: v_or_b32_e32 v3, v0, v1
-; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_mov_b32 s5, s4
; VI-NEXT: buffer_store_dwordx2 v[3:4], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1223,11 +1236,12 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 {
; GFX9-NEXT: v_perm_b32 v5, v6, v7, s4
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_perm_b32 v1, v2, v3, s4
+; GFX9-NEXT: s_mov_b32 s4, 0
; GFX9-NEXT: v_lshl_or_b32 v4, v5, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s4
; GFX9-NEXT: buffer_store_dwordx2 v[3:4], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1239,11 +1253,12 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 {
; GFX11-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX11-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
-; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: v_lshl_or_b32 v1, v5, 16, v4
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-NEXT: s_mov_b32 s1, s0
; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-NEXT: s_setpc_b64 s[30:31]
store <8 x i8> %arg0, ptr addrspace(1) null
@@ -1286,13 +1301,14 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; CI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; CI-NEXT: s_mov_b32 s4, 0
; CI-NEXT: v_or_b32_e32 v12, v12, v14
; CI-NEXT: v_or_b32_e32 v11, v8, v10
; CI-NEXT: v_or_b32_e32 v10, v4, v6
; CI-NEXT: v_or_b32_e32 v9, v0, v2
-; CI-NEXT: s_mov_b64 s[4:5], 0
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: s_mov_b32 s5, s4
; CI-NEXT: buffer_store_dwordx4 v[9:12], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1313,13 +1329,14 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; VI-NEXT: s_mov_b32 s4, 0
; VI-NEXT: v_or_b32_e32 v12, v12, v13
; VI-NEXT: v_or_b32_e32 v11, v8, v9
; VI-NEXT: v_or_b32_e32 v10, v4, v5
; VI-NEXT: v_or_b32_e32 v9, v0, v1
-; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_mov_b32 s5, s4
; VI-NEXT: buffer_store_dwordx4 v[9:12], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1336,13 +1353,14 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; GFX9-NEXT: v_perm_b32 v5, v6, v7, s4
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_perm_b32 v1, v2, v3, s4
+; GFX9-NEXT: s_mov_b32 s4, 0
; GFX9-NEXT: v_lshl_or_b32 v12, v13, 16, v12
; GFX9-NEXT: v_lshl_or_b32 v11, v9, 16, v8
; GFX9-NEXT: v_lshl_or_b32 v10, v5, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v9, v1, 16, v0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s4
; GFX9-NEXT: buffer_store_dwordx4 v[9:12], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1362,10 +1380,11 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, s0
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1383,10 +1402,11 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, s0
; GFX11-FAKE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
store <16 x i8> %arg0, ptr addrspace(1) null
@@ -1448,7 +1468,8 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; CI-NEXT: v_or_b32_e32 v2, v8, v9
; CI-NEXT: v_and_b32_e32 v8, 0xff, v20
; CI-NEXT: v_and_b32_e32 v9, 0xff, v16
-; CI-NEXT: s_mov_b64 s[4:5], 16
+; CI-NEXT: s_mov_b32 s5, 0
+; CI-NEXT: s_mov_b32 s4, 16
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -1475,7 +1496,7 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; CI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; CI-NEXT: v_or_b32_e32 v4, v8, v4
; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
-; CI-NEXT: s_mov_b64 s[4:5], 0
+; CI-NEXT: s_mov_b32 s4, s5
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: s_setpc_b64 s[30:31]
@@ -1506,7 +1527,8 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v3
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; VI-NEXT: s_mov_b64 s[4:5], 16
+; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_mov_b32 s4, 16
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
@@ -1522,7 +1544,7 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; VI-NEXT: v_or_b32_e32 v7, v7, v8
; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_mov_b32 s4, s5
; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
@@ -1548,7 +1570,8 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX9-NEXT: v_perm_b32 v16, v16, v17, s8
; GFX9-NEXT: v_perm_b32 v17, v18, v19, s8
; GFX9-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX9-NEXT: s_mov_b64 s[4:5], 16
+; GFX9-NEXT: s_mov_b32 s5, 0
+; GFX9-NEXT: s_mov_b32 s4, 16
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v4
@@ -1561,7 +1584,7 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX9-NEXT: v_perm_b32 v8, v30, v10, s8
; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX9-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_mov_b32 s4, s5
; GFX9-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -1592,7 +1615,8 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v11, 16, v10
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v14, 16, v0
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 16
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 16
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -1600,7 +1624,7 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v17
; GFX11-TRUE16-NEXT: buffer_store_b128 v[4:7], off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s1
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1629,7 +1653,8 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v17, 16, v16
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v14, 16, v0
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 16
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 16
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -1638,7 +1663,7 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v12, 16, v1
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v11, 16, v10
; GFX11-FAKE16-NEXT: buffer_store_b128 v[4:7], off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s1
; GFX11-FAKE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
store <32 x i8> %arg0, ptr addrspace(1) null
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index c06011c259f9b..eb47491beb62f 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -3282,11 +3282,11 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v42, v2
; GFX11-NEXT: v_mov_b32_e32 v51, v24
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:528
; GFX11-NEXT: v_mov_b32_e32 v52, v25
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:544
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1568 ; 16-byte Folded Spill
@@ -3317,8 +3317,8 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: v_mov_b32_e32 v29, v43
; GFX11-NEXT: scratch_store_b128 off, v[50:53], s2
; GFX11-NEXT: s_clause 0x3 ; 64-byte Folded Reload
-; GFX11-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1584
-; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536
+; GFX11-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1536
+; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1584
; GFX11-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1568
; GFX11-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1552
; GFX11-NEXT: s_add_i32 s2, s33, 0x400
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index a090a1b804677..16b6211fd5d3c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -321,14 +321,14 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GCN-SDAG-NEXT: s_wait_kmcnt 0x0
; GCN-SDAG-NEXT: s_clause 0x7
-; GCN-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off offset:112
-; GCN-SDAG-NEXT: global_load_b128 v[10:13], v[0:1], off offset:96
-; GCN-SDAG-NEXT: global_load_b128 v[14:17], v[0:1], off offset:80
-; GCN-SDAG-NEXT: global_load_b128 v[18:21], v[0:1], off offset:48
-; GCN-SDAG-NEXT: global_load_b128 v[22:25], v[0:1], off offset:32
-; GCN-SDAG-NEXT: global_load_b128 v[26:29], v[0:1], off offset:16
-; GCN-SDAG-NEXT: global_load_b128 v[30:33], v[0:1], off
-; GCN-SDAG-NEXT: global_load_b128 v[34:37], v[0:1], off offset:64
+; GCN-SDAG-NEXT: global_load_b128 v[10:13], v[0:1], off offset:112
+; GCN-SDAG-NEXT: global_load_b128 v[14:17], v[0:1], off offset:96
+; GCN-SDAG-NEXT: global_load_b128 v[6:9], v[0:1], off offset:80
+; GCN-SDAG-NEXT: global_load_b128 v[22:25], v[0:1], off offset:48
+; GCN-SDAG-NEXT: global_load_b128 v[26:29], v[0:1], off offset:32
+; GCN-SDAG-NEXT: global_load_b128 v[30:33], v[0:1], off offset:16
+; GCN-SDAG-NEXT: global_load_b128 v[34:37], v[0:1], off
+; GCN-SDAG-NEXT: global_load_b128 v[18:21], v[0:1], off offset:64
; GCN-SDAG-NEXT: v_mov_b64_e32 v[48:49], 48
; GCN-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x70
; GCN-SDAG-NEXT: v_mov_b64_e32 v[50:51], 32
@@ -340,53 +340,53 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, 0xc8
; GCN-SDAG-NEXT: v_mov_b64_e32 v[54:55], 64
; GCN-SDAG-NEXT: s_wait_loadcnt 0x7
-; GCN-SDAG-NEXT: global_store_b128 v[2:3], v[6:9], off
+; GCN-SDAG-NEXT: global_store_b128 v[2:3], v[10:13], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x6
-; GCN-SDAG-NEXT: global_store_b128 v[38:39], v[10:13], off
+; GCN-SDAG-NEXT: global_store_b128 v[38:39], v[14:17], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x5
; GCN-SDAG-NEXT: s_wait_xcnt 0x1
-; GCN-SDAG-NEXT: v_dual_mov_b32 v2, v16 :: v_dual_mov_b32 v3, v17
+; GCN-SDAG-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GCN-SDAG-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[14:15], v[14:15], v[14:15]
; GCN-SDAG-NEXT: s_wait_loadcnt 0x4
-; GCN-SDAG-NEXT: global_store_b128 v[48:49], v[18:21], off
+; GCN-SDAG-NEXT: global_store_b128 v[48:49], v[22:25], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x3
-; GCN-SDAG-NEXT: global_store_b128 v[50:51], v[22:25], off
+; GCN-SDAG-NEXT: global_store_b128 v[50:51], v[26:29], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x2
-; GCN-SDAG-NEXT: global_store_b128 v[64:65], v[26:29], off
+; GCN-SDAG-NEXT: global_store_b128 v[64:65], v[30:33], off
; GCN-SDAG-NEXT: s_wait_loadcnt 0x1
-; GCN-SDAG-NEXT: global_store_b128 v[66:67], v[30:33], off
+; GCN-SDAG-NEXT: global_store_b128 v[66:67], v[34:37], off
; GCN-SDAG-NEXT: s_wait_xcnt 0x0
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[6:7]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[36:37], v[36:37], v[36:37]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
; GCN-SDAG-NEXT: s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[50:51], v[36:37], v[36:37]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[48:49], v[34:35], v[34:35]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[14:15], 0xc8, v[14:15]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[28:29], v[28:29], v[28:29]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[26:27], v[26:27], v[26:27]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[50:51], v[20:21], v[20:21]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[48:49], v[18:19], v[18:19]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], 0xc8, v[6:7]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[30:31], v[30:31], v[30:31]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[20:21], v[20:21], v[20:21]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[18:19], v[18:19], v[18:19]
-; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[24:25], 0x64, v[24:25]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[34:35], v[34:35], v[34:35]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[24:25], v[24:25], v[24:25]
; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[22:23], v[22:23], v[22:23]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[28:29], 0x64, v[28:29]
+; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[26:27], v[26:27], v[26:27]
; GCN-SDAG-NEXT: s_clause 0x1
; GCN-SDAG-NEXT: global_store_b128 v[52:53], v[0:3], off
-; GCN-SDAG-NEXT: global_store_b128 v[54:55], v[34:37], off
+; GCN-SDAG-NEXT: global_store_b128 v[54:55], v[18:21], off
; GCN-SDAG-NEXT: s_clause 0x7
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[10:13], off offset:96
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[6:9], off offset:112
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[14:17], off offset:96
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[10:13], off offset:112
; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[48:51], off offset:64
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[14:17], off offset:80
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[22:25], off offset:32
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[18:21], off offset:48
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[30:33], off
-; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[26:29], off offset:16
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[6:9], off offset:80
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[26:29], off offset:32
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[22:25], off offset:48
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[34:37], off
+; GCN-SDAG-NEXT: global_store_b128 v[4:5], v[30:33], off offset:16
; GCN-SDAG-NEXT: s_wait_xcnt 0x9
-; GCN-SDAG-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GCN-SDAG-NEXT: v_dual_mov_b32 v0, v36 :: v_dual_mov_b32 v1, v37
; GCN-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-GISEL-LABEL: test_v16i64_load_store:
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index 76f204dd0c16a..15e792f806950 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -35,16 +35,16 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_mov_b32 s69, s4
; CHECK-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
; CHECK-NEXT: s_load_dwordx8 s[24:31], s[68:69], 0x30
-; CHECK-NEXT: s_load_dwordx16 s[52:67], s[68:69], 0xf0
+; CHECK-NEXT: s_load_dwordx16 s[36:51], s[68:69], 0xf0
; CHECK-NEXT: ; kill: killed $sgpr8_sgpr9
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: s_load_dwordx16 s[8:23], s[68:69], 0x130
; CHECK-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane
+; CHECK-NEXT: s_load_dwordx16 s[52:67], s[68:69], 0x2f0
; CHECK-NEXT: v_writelane_b32 v6, s70, 20
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_writelane_b32 v6, s71, 21
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v1, s4
-; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_writelane_b32 v7, s8, 0
; CHECK-NEXT: v_writelane_b32 v7, s9, 1
; CHECK-NEXT: v_writelane_b32 v7, s10, 2
@@ -76,14 +76,29 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_writelane_b32 v7, s64, 28
; CHECK-NEXT: v_writelane_b32 v7, s65, 29
; CHECK-NEXT: v_writelane_b32 v7, s66, 30
+; CHECK-NEXT: v_writelane_b32 v7, s67, 31
+; CHECK-NEXT: v_writelane_b32 v7, s36, 32
+; CHECK-NEXT: v_writelane_b32 v7, s37, 33
+; CHECK-NEXT: v_writelane_b32 v7, s38, 34
+; CHECK-NEXT: v_writelane_b32 v7, s39, 35
+; CHECK-NEXT: v_writelane_b32 v7, s40, 36
+; CHECK-NEXT: v_writelane_b32 v7, s41, 37
+; CHECK-NEXT: v_writelane_b32 v7, s42, 38
+; CHECK-NEXT: v_writelane_b32 v7, s43, 39
+; CHECK-NEXT: v_writelane_b32 v7, s44, 40
+; CHECK-NEXT: v_writelane_b32 v7, s45, 41
+; CHECK-NEXT: v_writelane_b32 v7, s46, 42
+; CHECK-NEXT: v_writelane_b32 v7, s47, 43
+; CHECK-NEXT: v_writelane_b32 v7, s48, 44
+; CHECK-NEXT: v_writelane_b32 v7, s49, 45
+; CHECK-NEXT: v_writelane_b32 v7, s50, 46
+; CHECK-NEXT: v_writelane_b32 v7, s51, 47
+; CHECK-NEXT: v_readlane_b32 s52, v7, 0
; CHECK-NEXT: s_load_dwordx16 s[8:23], s[68:69], 0x1f0
-; CHECK-NEXT: s_load_dwordx16 s[36:51], s[68:69], 0x2f0
+; CHECK-NEXT: v_mov_b32_e32 v1, s4
; CHECK-NEXT: s_mov_b32 s69, s68
; CHECK-NEXT: s_mov_b32 s70, s68
; CHECK-NEXT: s_mov_b32 s71, s68
-; CHECK-NEXT: v_writelane_b32 v7, s67, 31
-; CHECK-NEXT: image_sample_lz v1, v[1:2], s[60:67], s[68:71] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s52, v7, 0
; CHECK-NEXT: v_mov_b32_e32 v3, v2
; CHECK-NEXT: v_readlane_b32 s53, v7, 1
; CHECK-NEXT: v_readlane_b32 s54, v7, 2
@@ -92,12 +107,29 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_readlane_b32 s57, v7, 5
; CHECK-NEXT: v_readlane_b32 s58, v7, 6
; CHECK-NEXT: v_readlane_b32 s59, v7, 7
+; CHECK-NEXT: image_sample_lz v1, v[1:2], s[44:51], s[68:71] dmask:0x1
+; CHECK-NEXT: v_readlane_b32 s36, v7, 16
; CHECK-NEXT: v_and_b32_e32 v5, 1, v0
+; CHECK-NEXT: v_readlane_b32 s37, v7, 17
+; CHECK-NEXT: v_readlane_b32 s38, v7, 18
+; CHECK-NEXT: image_sample_lz v4, v[2:3], s[52:59], s[68:71] dmask:0x1
+; CHECK-NEXT: v_readlane_b32 s39, v7, 19
+; CHECK-NEXT: v_readlane_b32 s40, v7, 20
+; CHECK-NEXT: v_readlane_b32 s41, v7, 21
+; CHECK-NEXT: v_readlane_b32 s42, v7, 22
+; CHECK-NEXT: v_readlane_b32 s43, v7, 23
+; CHECK-NEXT: v_readlane_b32 s44, v7, 24
+; CHECK-NEXT: v_readlane_b32 s45, v7, 25
+; CHECK-NEXT: v_readlane_b32 s46, v7, 26
+; CHECK-NEXT: v_readlane_b32 s47, v7, 27
+; CHECK-NEXT: v_readlane_b32 s48, v7, 28
+; CHECK-NEXT: v_readlane_b32 s49, v7, 29
+; CHECK-NEXT: v_readlane_b32 s50, v7, 30
+; CHECK-NEXT: v_readlane_b32 s51, v7, 31
; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v5
; CHECK-NEXT: v_readlane_b32 s60, v7, 8
; CHECK-NEXT: v_readlane_b32 s61, v7, 9
; CHECK-NEXT: v_readlane_b32 s62, v7, 10
-; CHECK-NEXT: image_sample_lz v4, v[2:3], s[52:59], s[68:71] dmask:0x1
; CHECK-NEXT: v_readlane_b32 s63, v7, 11
; CHECK-NEXT: v_readlane_b32 s64, v7, 12
; CHECK-NEXT: v_readlane_b32 s65, v7, 13
@@ -109,25 +141,25 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %bb48
-; CHECK-NEXT: v_readlane_b32 s52, v7, 16
-; CHECK-NEXT: v_readlane_b32 s60, v7, 24
-; CHECK-NEXT: v_readlane_b32 s61, v7, 25
-; CHECK-NEXT: v_readlane_b32 s62, v7, 26
-; CHECK-NEXT: v_readlane_b32 s63, v7, 27
-; CHECK-NEXT: v_readlane_b32 s64, v7, 28
-; CHECK-NEXT: v_readlane_b32 s65, v7, 29
-; CHECK-NEXT: v_readlane_b32 s66, v7, 30
-; CHECK-NEXT: v_readlane_b32 s67, v7, 31
+; CHECK-NEXT: v_readlane_b32 s52, v7, 32
+; CHECK-NEXT: v_readlane_b32 s60, v7, 40
+; CHECK-NEXT: v_readlane_b32 s61, v7, 41
+; CHECK-NEXT: v_readlane_b32 s62, v7, 42
+; CHECK-NEXT: v_readlane_b32 s63, v7, 43
+; CHECK-NEXT: v_readlane_b32 s64, v7, 44
+; CHECK-NEXT: v_readlane_b32 s65, v7, 45
+; CHECK-NEXT: v_readlane_b32 s66, v7, 46
+; CHECK-NEXT: v_readlane_b32 s67, v7, 47
; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_and_b64 vcc, exec, -1
-; CHECK-NEXT: v_readlane_b32 s53, v7, 17
-; CHECK-NEXT: v_readlane_b32 s54, v7, 18
-; CHECK-NEXT: v_readlane_b32 s55, v7, 19
+; CHECK-NEXT: v_readlane_b32 s53, v7, 33
+; CHECK-NEXT: v_readlane_b32 s54, v7, 34
+; CHECK-NEXT: v_readlane_b32 s55, v7, 35
; CHECK-NEXT: image_sample_lz v3, v[2:3], s[60:67], s[68:71] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s56, v7, 20
-; CHECK-NEXT: v_readlane_b32 s57, v7, 21
-; CHECK-NEXT: v_readlane_b32 s58, v7, 22
-; CHECK-NEXT: v_readlane_b32 s59, v7, 23
+; CHECK-NEXT: v_readlane_b32 s56, v7, 36
+; CHECK-NEXT: v_readlane_b32 s57, v7, 37
+; CHECK-NEXT: v_readlane_b32 s58, v7, 38
+; CHECK-NEXT: v_readlane_b32 s59, v7, 39
; CHECK-NEXT: .LBB0_2: ; %bb50
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
@@ -141,37 +173,37 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_mov_b64 vcc, vcc
; CHECK-NEXT: s_cbranch_vccnz .LBB0_2
; CHECK-NEXT: .LBB0_3: ; %Flow14
-; CHECK-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_andn2_saveexec_b64 s[16:17], s[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.4: ; %bb32
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_and_saveexec_b64 s[16:17], s[4:5]
-; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[16:17]
+; CHECK-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT: s_xor_b64 s[18:19], exec, s[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB0_6
; CHECK-NEXT: ; %bb.5: ; %bb43
-; CHECK-NEXT: s_mov_b32 s16, 0
-; CHECK-NEXT: s_mov_b32 s17, s16
-; CHECK-NEXT: v_mov_b32_e32 v0, s16
-; CHECK-NEXT: v_readlane_b32 s44, v7, 16
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
-; CHECK-NEXT: s_mov_b32 s18, s16
-; CHECK-NEXT: s_mov_b32 s19, s16
-; CHECK-NEXT: v_readlane_b32 s45, v7, 17
-; CHECK-NEXT: v_readlane_b32 s46, v7, 18
-; CHECK-NEXT: v_readlane_b32 s47, v7, 19
-; CHECK-NEXT: v_readlane_b32 s48, v7, 20
-; CHECK-NEXT: v_readlane_b32 s49, v7, 21
-; CHECK-NEXT: v_readlane_b32 s50, v7, 22
-; CHECK-NEXT: v_readlane_b32 s51, v7, 23
-; CHECK-NEXT: v_readlane_b32 s52, v7, 24
-; CHECK-NEXT: v_readlane_b32 s53, v7, 25
-; CHECK-NEXT: v_readlane_b32 s54, v7, 26
-; CHECK-NEXT: v_readlane_b32 s55, v7, 27
-; CHECK-NEXT: v_readlane_b32 s56, v7, 28
-; CHECK-NEXT: v_readlane_b32 s57, v7, 29
-; CHECK-NEXT: v_readlane_b32 s58, v7, 30
-; CHECK-NEXT: v_readlane_b32 s59, v7, 31
-; CHECK-NEXT: image_sample_lz v2, v[0:1], s[44:51], s[16:19] dmask:0x1
+; CHECK-NEXT: s_mov_b32 s4, 0
+; CHECK-NEXT: s_mov_b32 s5, s4
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
+; CHECK-NEXT: v_readlane_b32 s44, v7, 32
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: s_mov_b32 s6, s4
+; CHECK-NEXT: s_mov_b32 s7, s4
+; CHECK-NEXT: v_readlane_b32 s45, v7, 33
+; CHECK-NEXT: v_readlane_b32 s46, v7, 34
+; CHECK-NEXT: v_readlane_b32 s47, v7, 35
+; CHECK-NEXT: v_readlane_b32 s48, v7, 36
+; CHECK-NEXT: v_readlane_b32 s49, v7, 37
+; CHECK-NEXT: v_readlane_b32 s50, v7, 38
+; CHECK-NEXT: v_readlane_b32 s51, v7, 39
+; CHECK-NEXT: v_readlane_b32 s52, v7, 40
+; CHECK-NEXT: v_readlane_b32 s53, v7, 41
+; CHECK-NEXT: v_readlane_b32 s54, v7, 42
+; CHECK-NEXT: v_readlane_b32 s55, v7, 43
+; CHECK-NEXT: v_readlane_b32 s56, v7, 44
+; CHECK-NEXT: v_readlane_b32 s57, v7, 45
+; CHECK-NEXT: v_readlane_b32 s58, v7, 46
+; CHECK-NEXT: v_readlane_b32 s59, v7, 47
+; CHECK-NEXT: image_sample_lz v2, v[0:1], s[44:51], s[4:7] dmask:0x1
; CHECK-NEXT: v_readlane_b32 s44, v7, 0
; CHECK-NEXT: v_readlane_b32 s52, v7, 8
; CHECK-NEXT: v_readlane_b32 s53, v7, 9
@@ -192,24 +224,24 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_readlane_b32 s50, v7, 6
; CHECK-NEXT: v_readlane_b32 s51, v7, 7
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[16:19], 0
+; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[4:7], 0
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: .LBB0_6: ; %Flow12
-; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[18:19]
; CHECK-NEXT: s_cbranch_execz .LBB0_9
; CHECK-NEXT: ; %bb.7: ; %bb33.preheader
-; CHECK-NEXT: s_mov_b32 s16, 0
-; CHECK-NEXT: s_mov_b32 s20, s16
-; CHECK-NEXT: s_mov_b32 s21, s16
-; CHECK-NEXT: v_mov_b32_e32 v1, s20
-; CHECK-NEXT: s_mov_b32 s17, s16
-; CHECK-NEXT: s_mov_b32 s18, s16
-; CHECK-NEXT: s_mov_b32 s19, s16
-; CHECK-NEXT: v_mov_b32_e32 v2, s21
-; CHECK-NEXT: image_sample_lz v3, v[1:2], s[8:15], s[16:19] dmask:0x1
-; CHECK-NEXT: image_sample_lz v4, v[1:2], s[36:43], s[16:19] dmask:0x1
+; CHECK-NEXT: s_mov_b32 s20, 0
+; CHECK-NEXT: s_mov_b32 s6, s20
+; CHECK-NEXT: s_mov_b32 s7, s20
+; CHECK-NEXT: v_mov_b32_e32 v1, s6
+; CHECK-NEXT: s_mov_b32 s21, s20
+; CHECK-NEXT: s_mov_b32 s22, s20
+; CHECK-NEXT: s_mov_b32 s23, s20
+; CHECK-NEXT: v_mov_b32_e32 v2, s7
+; CHECK-NEXT: image_sample_lz v3, v[1:2], s[8:15], s[20:23] dmask:0x1
+; CHECK-NEXT: image_sample_lz v4, v[1:2], s[36:43], s[20:23] dmask:0x1
; CHECK-NEXT: s_and_b64 vcc, exec, 0
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_sub_f32_e32 v1, v4, v3
@@ -224,7 +256,7 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: .LBB0_9: ; %Flow13
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: .LBB0_10: ; %UnifiedReturnBlock
-; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_or_b64 exec, exec, s[16:17]
; CHECK-NEXT: v_readlane_b32 s71, v6, 21
; CHECK-NEXT: v_readlane_b32 s70, v6, 20
; CHECK-NEXT: v_readlane_b32 s69, v6, 19
@@ -237,7 +269,6 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: v_readlane_b32 s54, v6, 12
; CHECK-NEXT: v_readlane_b32 s53, v6, 11
; CHECK-NEXT: v_readlane_b32 s52, v6, 10
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_readlane_b32 s51, v6, 9
; CHECK-NEXT: v_readlane_b32 s50, v6, 8
; CHECK-NEXT: v_readlane_b32 s49, v6, 7
diff --git a/llvm/test/CodeGen/AMDGPU/idot2.ll b/llvm/test/CodeGen/AMDGPU/idot2.ll
index de0e3f05e47fa..5f92df62b1364 100644
--- a/llvm/test/CodeGen/AMDGPU/idot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot2.ll
@@ -2248,20 +2248,20 @@ entry:
define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,
; GFX7-LABEL: udot2_MultipleUses_mul2:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -2271,9 +2271,9 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v4
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, s0, v1
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, s4, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot2_MultipleUses_mul2:
@@ -2396,20 +2396,20 @@ entry:
define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,
; GFX7-LABEL: idot2_MultipleUses_mul2:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16
; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2
@@ -2419,9 +2419,9 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,
; GFX7-NEXT: v_mul_i32_i24_e32 v4, v0, v2
; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, s0, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, s4, v0
; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot2_MultipleUses_mul2:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index 917dc51254396..e2a10b1e04552 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -400,20 +400,20 @@ entry:
define amdgpu_kernel void @idot4_acc8(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
@@ -429,7 +429,7 @@ define amdgpu_kernel void @idot4_acc8(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc8:
@@ -1420,20 +1420,20 @@ entry:
define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_3ele:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
@@ -1441,12 +1441,12 @@ define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,
; GFX7-NEXT: v_bfe_i32 v4, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 8
; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 8
; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_3ele:
@@ -1601,20 +1601,20 @@ entry:
define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_3ele_permuted:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_ashrrev_i32_e32 v1, 24, v2
; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
@@ -1622,12 +1622,12 @@ define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,
; GFX7-NEXT: v_ashrrev_i32_e32 v4, 24, v0
; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 8
; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 8
; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_3ele_permuted:
@@ -1781,19 +1781,19 @@ entry:
define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_opt:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
@@ -1808,7 +1808,7 @@ define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,
; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_opt:
@@ -2563,20 +2563,20 @@ entry:
define amdgpu_kernel void @idot4_commutative(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_commutative:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xf
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8
; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8
@@ -2584,12 +2584,12 @@ define amdgpu_kernel void @idot4_commutative(ptr addrspace(1) %src1,
; GFX7-NEXT: v_bfe_i32 v4, v0, 0, 8
; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 8
; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 8
; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_commutative:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b67c40b2d8333..eed019b8e75e4 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -190,20 +190,20 @@ entry:
define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_acc16:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ushort v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
@@ -219,7 +219,7 @@ define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_acc16:
@@ -386,20 +386,20 @@ entry:
define amdgpu_kernel void @udot4_acc8(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_acc8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
@@ -415,7 +415,7 @@ define amdgpu_kernel void @udot4_acc8(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_acc8:
@@ -728,20 +728,20 @@ entry:
define amdgpu_kernel void @udot4_CommutationInsideMAD(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_CommutationInsideMAD:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
@@ -757,7 +757,7 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
-; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_CommutationInsideMAD:
@@ -906,20 +906,20 @@ entry:
define amdgpu_kernel void @udot4_CommutationAccrossMADs(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_CommutationAccrossMADs:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
@@ -935,7 +935,7 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
-; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_CommutationAccrossMADs:
@@ -2522,20 +2522,20 @@ entry:
define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_acc8_vecMul:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 8
@@ -2551,7 +2551,7 @@ define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v0
-; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_acc8_vecMul:
@@ -2963,20 +2963,20 @@ entry:
define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_3ele:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
@@ -2984,12 +2984,12 @@ define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,
; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_3ele:
@@ -3142,20 +3142,20 @@ entry:
define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_3ele_permuted:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
@@ -3163,12 +3163,12 @@ define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 24, v0
; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_3ele_permuted:
@@ -3322,19 +3322,19 @@ entry:
define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_opt:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
@@ -3349,7 +3349,7 @@ define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_opt:
@@ -4104,20 +4104,20 @@ entry:
define amdgpu_kernel void @udot4_commutative(ptr addrspace(1) %src1,
; GFX7-LABEL: udot4_commutative:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xf
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
@@ -4125,12 +4125,12 @@ define amdgpu_kernel void @udot4_commutative(ptr addrspace(1) %src1,
; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s0
+; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: udot4_commutative:
@@ -5153,19 +5153,19 @@ entry:
define amdgpu_kernel void @idot4_acc32_lohi(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_lohi:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:4
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:4
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
@@ -5180,7 +5180,7 @@ define amdgpu_kernel void @idot4_acc32_lohi(ptr addrspace(1) %src1,
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_lohi:
@@ -5333,19 +5333,19 @@ entry:
define amdgpu_kernel void @idot4_acc32_hihi(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_hihi:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
-; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:4
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:4
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
+; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 offset:4
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 offset:4
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_u32 v3, v2, 16, 8
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
@@ -5360,7 +5360,7 @@ define amdgpu_kernel void @idot4_acc32_hihi(ptr addrspace(1) %src1,
; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_hihi:
@@ -5667,22 +5667,22 @@ entry:
define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_v16i8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]
+; GFX7-NEXT: s_mov_b64 s[8:9], s[10:11]
+; GFX7-NEXT: s_mov_b64 s[10:11], s[6:7]
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v5, v2
-; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[1:2], s[8:11], 0 addr64
-; GFX7-NEXT: buffer_load_dword v0, v[4:5], s[0:3], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[1:2], s[4:7], 0 addr64
+; GFX7-NEXT: buffer_load_dword v0, v[4:5], s[8:11], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
@@ -5697,7 +5697,7 @@ define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX7-NEXT: v_mad_u32_u24 v1, v6, v7, v1
; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_v16i8:
@@ -5862,21 +5862,21 @@ entry:
define amdgpu_kernel void @idot4_acc32_v256i8(ptr addrspace(1) %src1,
; GFX7-LABEL: idot4_acc32_v256i8:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, s7
+; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, s3
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]
+; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]
+; GFX7-NEXT: s_mov_b64 s[10:11], s[6:7]
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_load_dword v0, v[1:2], s[0:3], 0 addr64 offset:252
-; GFX7-NEXT: buffer_load_dword v1, v[3:4], s[8:11], 0 addr64
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: buffer_load_dword v0, v[1:2], s[8:11], 0 addr64 offset:252
+; GFX7-NEXT: buffer_load_dword v1, v[3:4], s[4:7], 0 addr64
+; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_bfe_u32 v4, v0, 16, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -5891,7 +5891,7 @@ define amdgpu_kernel void @idot4_acc32_v256i8(ptr addrspace(1) %src1,
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX7-NEXT: v_mad_u32_u24 v2, v6, v7, v2
; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, v2
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: idot4_acc32_v256i8:
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index 231520d148884..e891dc9ce866a 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -2538,19 +2538,19 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
;
; GFX9-LABEL: udot8_acc8_vecMul:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX9-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX9-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-NEXT: s_add_u32 s12, s12, s11
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[8:9]
+; GFX9-NEXT: global_load_dword v2, v0, s[10:11]
+; GFX9-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-NEXT: s_addc_u32 s13, s13, 0
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_bfe_u32 v0, v1, 4, 4
@@ -2598,24 +2598,24 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-NEXT: v_add_u16_e32 v0, v0, v8
; GFX9-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
; GFX9-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-NEXT: global_store_byte v3, v0, s[6:7]
+; GFX9-NEXT: global_store_byte v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX9-DL-LABEL: udot8_acc8_vecMul:
; GFX9-DL: ; %bb.0: ; %entry
-; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]
-; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX9-DL-NEXT: s_mov_b32 s14, -1
; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-DL-NEXT: s_add_u32 s12, s12, s11
+; GFX9-DL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-DL-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]
+; GFX9-DL-NEXT: global_load_dword v2, v0, s[10:11]
+; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[0:1]
; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 4, 4
@@ -2663,7 +2663,7 @@ define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v8
; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v16, v0
; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v7
-; GFX9-DL-NEXT: global_store_byte v3, v0, s[6:7]
+; GFX9-DL-NEXT: global_store_byte v3, v0, s[0:1]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: udot8_acc8_vecMul:
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index a9e67ff3fdcbb..7f243ea60e6d1 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -8230,28 +8230,28 @@ define amdgpu_kernel void @extract_largest_inbounds_offset(ptr addrspace(1) %out
;
; SI-MOVREL-LABEL: extract_largest_inbounds_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
; SI-MOVREL-NEXT: s_load_dword s12, s[4:5], 0xd
-; SI-MOVREL-NEXT: s_mov_b32 s7, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s6, -1
-; SI-MOVREL-NEXT: s_mov_b32 s10, s6
+; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s6, s2
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_mov_b32 s8, s2
-; SI-MOVREL-NEXT: s_mov_b32 s9, s3
-; SI-MOVREL-NEXT: s_mov_b32 s11, s7
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 glc
+; SI-MOVREL-NEXT: s_mov_b32 s4, s10
+; SI-MOVREL-NEXT: s_mov_b32 s5, s11
+; SI-MOVREL-NEXT: s_mov_b32 s7, s3
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[4:7], 0 offset:16 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:32 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[4:7], 0 offset:48 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
; SI-MOVREL-NEXT: s_add_i32 m0, s12, 15
-; SI-MOVREL-NEXT: s_mov_b32 s4, s0
-; SI-MOVREL-NEXT: s_mov_b32 s5, s1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s8
+; SI-MOVREL-NEXT: s_mov_b32 s1, s9
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extract_largest_inbounds_offset:
@@ -8537,28 +8537,28 @@ define amdgpu_kernel void @extract_out_of_bounds_offset(ptr addrspace(1) %out, p
;
; SI-MOVREL-LABEL: extract_out_of_bounds_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
; SI-MOVREL-NEXT: s_load_dword s12, s[4:5], 0xd
-; SI-MOVREL-NEXT: s_mov_b32 s7, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s6, -1
-; SI-MOVREL-NEXT: s_mov_b32 s10, s6
+; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s6, s2
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_mov_b32 s8, s2
-; SI-MOVREL-NEXT: s_mov_b32 s9, s3
-; SI-MOVREL-NEXT: s_mov_b32 s11, s7
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 glc
+; SI-MOVREL-NEXT: s_mov_b32 s4, s10
+; SI-MOVREL-NEXT: s_mov_b32 s5, s11
+; SI-MOVREL-NEXT: s_mov_b32 s7, s3
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[4:7], 0 offset:16 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:32 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[4:7], 0 offset:48 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
; SI-MOVREL-NEXT: s_add_i32 m0, s12, 16
-; SI-MOVREL-NEXT: s_mov_b32 s4, s0
-; SI-MOVREL-NEXT: s_mov_b32 s5, s1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s8
+; SI-MOVREL-NEXT: s_mov_b32 s1, s9
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extract_out_of_bounds_offset:
@@ -8845,28 +8845,28 @@ define amdgpu_kernel void @extractelement_v16i32_or_index(ptr addrspace(1) %out,
;
; SI-MOVREL-LABEL: extractelement_v16i32_or_index:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
; SI-MOVREL-NEXT: s_load_dword s12, s[4:5], 0xd
-; SI-MOVREL-NEXT: s_mov_b32 s7, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s6, -1
-; SI-MOVREL-NEXT: s_mov_b32 s10, s6
+; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s6, s2
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_mov_b32 s8, s2
-; SI-MOVREL-NEXT: s_mov_b32 s9, s3
-; SI-MOVREL-NEXT: s_mov_b32 s11, s7
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 glc
+; SI-MOVREL-NEXT: s_mov_b32 s4, s10
+; SI-MOVREL-NEXT: s_mov_b32 s5, s11
+; SI-MOVREL-NEXT: s_mov_b32 s7, s3
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[4:7], off, s[4:7], 0 offset:16 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[8:11], off, s[4:7], 0 offset:32 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
-; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:48 glc
+; SI-MOVREL-NEXT: buffer_load_dwordx4 v[12:15], off, s[4:7], 0 offset:48 glc
; SI-MOVREL-NEXT: s_waitcnt vmcnt(0)
; SI-MOVREL-NEXT: s_lshl_b32 m0, s12, 2
-; SI-MOVREL-NEXT: s_mov_b32 s4, s0
-; SI-MOVREL-NEXT: s_mov_b32 s5, s1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s8
+; SI-MOVREL-NEXT: s_mov_b32 s1, s9
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v1
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extractelement_v16i32_or_index:
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 7cbf9aeacfe48..dd3173db49b22 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -677,25 +677,25 @@ define amdgpu_kernel void @dynamic_insertelement_v8f32(ptr addrspace(1) %out, <8
;
; VI-LABEL: dynamic_insertelement_v8f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; VI-NEXT: s_load_dword s8, s[8:9], 0x40
+; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x20
+; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dword s4, s[8:9], 0x40
; VI-NEXT: v_mov_b32_e32 v8, 0x40a00000
-; VI-NEXT: s_mov_b32 s15, 0x1100f000
-; VI-NEXT: s_mov_b32 s14, -1
+; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v7, s7
-; VI-NEXT: s_mov_b32 m0, s8
+; VI-NEXT: v_mov_b32_e32 v0, s12
+; VI-NEXT: v_mov_b32_e32 v1, s13
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v6, s18
+; VI-NEXT: v_mov_b32_e32 v7, s19
+; VI-NEXT: s_mov_b32 m0, s4
; VI-NEXT: v_movreld_b32_e32 v0, v8
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <8 x float> %a, float 5.000000e+00, i32 %b
store <8 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -705,55 +705,54 @@ define amdgpu_kernel void @dynamic_insertelement_v8f32(ptr addrspace(1) %out, <8
define amdgpu_kernel void @dynamic_insertelement_v9f32(ptr addrspace(1) %out, <9 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v9f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
-; SI-NEXT: s_load_dword s10, s[8:9], 0x18
-; SI-NEXT: s_load_dword s8, s[8:9], 0x20
+; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
+; SI-NEXT: s_load_dword s4, s[8:9], 0x18
+; SI-NEXT: s_load_dword s5, s[8:9], 0x20
; SI-NEXT: v_mov_b32_e32 v9, 0x40a00000
-; SI-NEXT: s_mov_b32 s15, 0x100f000
+; SI-NEXT: s_mov_b32 s3, 0x100f000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_mov_b32_e32 v3, s3
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_mov_b32_e32 v5, s5
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s10
-; SI-NEXT: s_mov_b32 m0, s8
-; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: s_mov_b32 m0, s5
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_movreld_b32_e32 v0, v9
-; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v9f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; VI-NEXT: s_load_dword s10, s[8:9], 0x60
+; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
+; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dword s4, s[8:9], 0x60
+; VI-NEXT: s_load_dword s5, s[8:9], 0x80
; VI-NEXT: v_mov_b32_e32 v9, 0x40a00000
-; VI-NEXT: s_mov_b32 s15, 0x1100f000
-; VI-NEXT: s_mov_b32 s14, -1
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_load_dword s0, s[8:9], 0x80
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v7, s7
-; VI-NEXT: v_mov_b32_e32 v8, s10
+; VI-NEXT: s_mov_b32 s3, 0x1100f000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 m0, s0
+; VI-NEXT: v_mov_b32_e32 v0, s12
+; VI-NEXT: v_mov_b32_e32 v1, s13
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v6, s18
+; VI-NEXT: v_mov_b32_e32 v7, s19
+; VI-NEXT: v_mov_b32_e32 v8, s4
+; VI-NEXT: s_mov_b32 m0, s5
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_movreld_b32_e32 v0, v9
-; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dword v8, off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <9 x float> %a, float 5.000000e+00, i32 %b
store <9 x float> %vecins, ptr addrspace(1) %out, align 32
@@ -763,29 +762,29 @@ define amdgpu_kernel void @dynamic_insertelement_v9f32(ptr addrspace(1) %out, <9
define amdgpu_kernel void @dynamic_insertelement_v10f32(ptr addrspace(1) %out, <10 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v10f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x18
-; SI-NEXT: s_load_dword s8, s[8:9], 0x20
+; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
+; SI-NEXT: s_load_dword s6, s[8:9], 0x20
; SI-NEXT: v_mov_b32_e32 v10, 0x40a00000
-; SI-NEXT: s_mov_b32 s15, 0x100f000
+; SI-NEXT: s_mov_b32 s3, 0x100f000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_mov_b32_e32 v3, s3
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_mov_b32_e32 v5, s5
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s10
-; SI-NEXT: v_mov_b32_e32 v9, s11
-; SI-NEXT: s_mov_b32 m0, s8
-; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: v_mov_b32_e32 v9, s5
+; SI-NEXT: s_mov_b32 m0, s6
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_movreld_b32_e32 v0, v10
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
-; SI-NEXT: buffer_store_dwordx2 v[8:9], off, s[12:15], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v10f32:
@@ -1176,24 +1175,24 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(ptr addrspace(1) %out, <8
;
; VI-LABEL: dynamic_insertelement_v8i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
-; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; VI-NEXT: s_load_dword s8, s[8:9], 0x40
-; VI-NEXT: s_mov_b32 s15, 0x1100f000
-; VI-NEXT: s_mov_b32 s14, -1
+; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x20
+; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dword s4, s[8:9], 0x40
+; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v7, s7
-; VI-NEXT: s_mov_b32 m0, s8
+; VI-NEXT: v_mov_b32_e32 v0, s12
+; VI-NEXT: v_mov_b32_e32 v1, s13
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v6, s18
+; VI-NEXT: v_mov_b32_e32 v7, s19
+; VI-NEXT: s_mov_b32 m0, s4
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <8 x i32> %a, i32 5, i32 %b
store <8 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1203,53 +1202,52 @@ define amdgpu_kernel void @dynamic_insertelement_v8i32(ptr addrspace(1) %out, <8
define amdgpu_kernel void @dynamic_insertelement_v9i32(ptr addrspace(1) %out, <9 x i32> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v9i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
-; SI-NEXT: s_load_dword s10, s[8:9], 0x18
-; SI-NEXT: s_load_dword s8, s[8:9], 0x20
-; SI-NEXT: s_mov_b32 s15, 0x100f000
-; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
+; SI-NEXT: s_load_dword s4, s[8:9], 0x18
+; SI-NEXT: s_load_dword s5, s[8:9], 0x20
+; SI-NEXT: s_mov_b32 s3, 0x100f000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_mov_b32_e32 v3, s3
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_mov_b32_e32 v5, s5
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s10
-; SI-NEXT: s_mov_b32 m0, s8
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: s_mov_b32 m0, s5
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v9i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; VI-NEXT: s_mov_b32 s15, 0x1100f000
-; VI-NEXT: s_mov_b32 s14, -1
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_load_dword s0, s[8:9], 0x60
-; VI-NEXT: s_load_dword s1, s[8:9], 0x80
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v7, s7
+; VI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x40
+; VI-NEXT: s_load_dword s4, s[8:9], 0x60
+; VI-NEXT: s_load_dword s5, s[8:9], 0x80
+; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v8, s0
-; VI-NEXT: s_mov_b32 m0, s1
+; VI-NEXT: v_mov_b32_e32 v0, s12
+; VI-NEXT: v_mov_b32_e32 v1, s13
+; VI-NEXT: v_mov_b32_e32 v2, s14
+; VI-NEXT: v_mov_b32_e32 v3, s15
+; VI-NEXT: v_mov_b32_e32 v4, s16
+; VI-NEXT: v_mov_b32_e32 v5, s17
+; VI-NEXT: v_mov_b32_e32 v6, s18
+; VI-NEXT: v_mov_b32_e32 v7, s19
+; VI-NEXT: v_mov_b32_e32 v8, s4
+; VI-NEXT: s_mov_b32 m0, s5
; VI-NEXT: v_movreld_b32_e32 v0, 5
-; VI-NEXT: buffer_store_dword v8, off, s[12:15], 0 offset:32
-; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; VI-NEXT: buffer_store_dword v8, off, s[0:3], 0 offset:32
+; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <9 x i32> %a, i32 5, i32 %b
store <9 x i32> %vecins, ptr addrspace(1) %out, align 32
@@ -1259,28 +1257,28 @@ define amdgpu_kernel void @dynamic_insertelement_v9i32(ptr addrspace(1) %out, <9
define amdgpu_kernel void @dynamic_insertelement_v10i32(ptr addrspace(1) %out, <10 x i32> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v10i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[12:13], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
-; SI-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x18
-; SI-NEXT: s_load_dword s8, s[8:9], 0x20
-; SI-NEXT: s_mov_b32 s15, 0x100f000
-; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x10
+; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
+; SI-NEXT: s_load_dword s6, s[8:9], 0x20
+; SI-NEXT: s_mov_b32 s3, 0x100f000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_mov_b32_e32 v3, s3
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_mov_b32_e32 v5, s5
-; SI-NEXT: v_mov_b32_e32 v6, s6
-; SI-NEXT: v_mov_b32_e32 v7, s7
-; SI-NEXT: v_mov_b32_e32 v8, s10
-; SI-NEXT: v_mov_b32_e32 v9, s11
-; SI-NEXT: s_mov_b32 m0, s8
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: v_mov_b32_e32 v3, s15
+; SI-NEXT: v_mov_b32_e32 v4, s16
+; SI-NEXT: v_mov_b32_e32 v5, s17
+; SI-NEXT: v_mov_b32_e32 v6, s18
+; SI-NEXT: v_mov_b32_e32 v7, s19
+; SI-NEXT: v_mov_b32_e32 v8, s4
+; SI-NEXT: v_mov_b32_e32 v9, s5
+; SI-NEXT: s_mov_b32 m0, s6
; SI-NEXT: v_movreld_b32_e32 v0, 5
-; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[12:15], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
-; SI-NEXT: buffer_store_dwordx2 v[8:9], off, s[12:15], 0 offset:32
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 0 offset:32
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v10i32:
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 6a5f2182961ba..0a4701517087d 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -5066,9 +5066,10 @@ define amdgpu_kernel void @struct_argument_alignment({i32, i64} %arg0, i8, {i32,
; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xb
; SI-NEXT: s_load_dword s9, s[4:5], 0xf
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x11
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -5204,7 +5205,8 @@ define amdgpu_kernel void @packed_struct_argument_alignment(<{i32, i64}> %arg0,
; SI-NEXT: buffer_load_ubyte v6, off, s[4:7], 0 offset:51
; SI-NEXT: buffer_load_ubyte v7, off, s[4:7], 0 offset:52
; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:53
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_mov_b32 s4, 0
+; SI-NEXT: s_mov_b32 s5, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: buffer_store_dword v2, off, s[4:7], 0
@@ -5382,9 +5384,10 @@ define amdgpu_kernel void @struct_argument_alignment_after({i32, i64} %arg0, i8,
; SI-NEXT: s_load_dword s13, s[4:5], 0xf
; SI-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x11
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x15
-; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s5, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s12
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
index 6be517f1e2116..133a2b89bb1d4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
@@ -6,24 +6,24 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: ds_read_b128 v[30:33], v0 offset:112
-; GCN-NEXT: ds_read_b128 v[26:29], v0 offset:96
+; GCN-NEXT: ds_read_b128 v[14:17], v0 offset:112
+; GCN-NEXT: ds_read_b128 v[18:21], v0 offset:96
; GCN-NEXT: ds_read_b128 v[22:25], v0 offset:80
-; GCN-NEXT: ds_read_b128 v[18:21], v0 offset:64
+; GCN-NEXT: ds_read_b128 v[26:29], v0 offset:64
; GCN-NEXT: ds_read_b128 v[2:5], v0
; GCN-NEXT: ds_read_b128 v[6:9], v0 offset:16
; GCN-NEXT: ds_read_b128 v[10:13], v0 offset:32
-; GCN-NEXT: ds_read_b128 v[14:17], v0 offset:48
+; GCN-NEXT: ds_read_b128 v[30:33], v0 offset:48
; GCN-NEXT: v_mov_b32_e32 v34, 0
; GCN-NEXT: v_mov_b32_e32 v35, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u64 s[0:1], 0
; GCN-NEXT: ; iglp_opt mask(0x00000001)
-; GCN-NEXT: ds_write_b128 v0, v[30:33] offset:112
-; GCN-NEXT: ds_write_b128 v0, v[26:29] offset:96
+; GCN-NEXT: ds_write_b128 v0, v[14:17] offset:112
+; GCN-NEXT: ds_write_b128 v0, v[18:21] offset:96
; GCN-NEXT: ds_write_b128 v0, v[22:25] offset:80
-; GCN-NEXT: ds_write_b128 v0, v[18:21] offset:64
-; GCN-NEXT: ds_write_b128 v0, v[14:17] offset:48
+; GCN-NEXT: ds_write_b128 v0, v[26:29] offset:64
+; GCN-NEXT: ds_write_b128 v0, v[30:33] offset:48
; GCN-NEXT: ds_write_b128 v0, v[10:13] offset:32
; GCN-NEXT: ds_write_b128 v0, v[6:9] offset:16
; GCN-NEXT: ds_write_b128 v0, v[2:5]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index a1fe463de1c54..20779d16dc4e9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -2863,60 +2863,56 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v40, s28
; SDAG-NEXT: v_mov_b32_e32 v41, s29
; SDAG-NEXT: v_mov_b32_e32 v42, s30
; SDAG-NEXT: v_mov_b32_e32 v43, s31
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[36:39], v[40:43], v[0:15]
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16
-; SDAG-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], 16
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], 0
+; SDAG-NEXT: v_mov_b32_e32 v4, s16
+; SDAG-NEXT: v_mov_b32_e32 v5, s17
+; SDAG-NEXT: v_mov_b32_e32 v6, s18
+; SDAG-NEXT: v_mov_b32_e32 v7, s19
+; SDAG-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], 0
-; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s16
-; SDAG-NEXT: v_mov_b32_e32 v1, s17
-; SDAG-NEXT: v_mov_b32_e32 v2, s18
-; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v4, s8
+; SDAG-NEXT: v_mov_b32_e32 v5, s9
+; SDAG-NEXT: v_mov_b32_e32 v6, s10
+; SDAG-NEXT: v_mov_b32_e32 v7, s11
+; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v2, s12
+; SDAG-NEXT: v_mov_b32_e32 v3, s13
+; SDAG-NEXT: v_mov_b32_e32 v4, s14
+; SDAG-NEXT: v_mov_b32_e32 v5, s15
+; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -2982,28 +2978,28 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b32_e32 v5, s25
; HEURRC-NEXT: v_mov_b32_e32 v6, s26
; HEURRC-NEXT: v_mov_b32_e32 v7, s27
-; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_mov_b32_e32 v8, s28
; HEURRC-NEXT: v_mov_b32_e32 v9, s29
; HEURRC-NEXT: v_mov_b32_e32 v10, s30
; HEURRC-NEXT: v_mov_b32_e32 v11, s31
-; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[4:7], v[8:11], a[0:15]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3011,13 +3007,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
@@ -3053,60 +3049,56 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[36:39], v[40:43], v[0:15]
; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
-; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s19
+; VGPRRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 0
-; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s23
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s11
+; VGPRRC-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s15
+; VGPRRC-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8:
@@ -3264,60 +3256,56 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
-; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v40, s28
; SDAG-NEXT: v_mov_b32_e32 v41, s29
; SDAG-NEXT: v_mov_b32_e32 v42, s30
; SDAG-NEXT: v_mov_b32_e32 v43, s31
-; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
-; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[36:39], v[40:43], v[0:15] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: s_nop 11
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b64_e32 v[8:9], 16
-; SDAG-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
+; SDAG-NEXT: v_mov_b64_e32 v[0:1], 16
+; SDAG-NEXT: v_mov_b64_e32 v[2:3], 0
+; SDAG-NEXT: v_mov_b32_e32 v4, s16
+; SDAG-NEXT: v_mov_b32_e32 v5, s17
+; SDAG-NEXT: v_mov_b32_e32 v6, s18
+; SDAG-NEXT: v_mov_b32_e32 v7, s19
+; SDAG-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b64_e32 v[4:5], 0
-; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s16
-; SDAG-NEXT: v_mov_b32_e32 v1, s17
-; SDAG-NEXT: v_mov_b32_e32 v2, s18
-; SDAG-NEXT: v_mov_b32_e32 v3, s19
-; SDAG-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; SDAG-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s20
-; SDAG-NEXT: v_mov_b32_e32 v1, s21
-; SDAG-NEXT: v_mov_b32_e32 v2, s22
-; SDAG-NEXT: v_mov_b32_e32 v3, s23
-; SDAG-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v4, s20
+; SDAG-NEXT: v_mov_b32_e32 v5, s21
+; SDAG-NEXT: v_mov_b32_e32 v6, s22
+; SDAG-NEXT: v_mov_b32_e32 v7, s23
+; SDAG-NEXT: global_store_dwordx4 v[32:33], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s8
-; SDAG-NEXT: v_mov_b32_e32 v1, s9
-; SDAG-NEXT: v_mov_b32_e32 v2, s10
-; SDAG-NEXT: v_mov_b32_e32 v3, s11
-; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v4, s8
+; SDAG-NEXT: v_mov_b32_e32 v5, s9
+; SDAG-NEXT: v_mov_b32_e32 v6, s10
+; SDAG-NEXT: v_mov_b32_e32 v7, s11
+; SDAG-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_nop 0
-; SDAG-NEXT: v_mov_b32_e32 v0, s12
-; SDAG-NEXT: v_mov_b32_e32 v1, s13
-; SDAG-NEXT: v_mov_b32_e32 v2, s14
-; SDAG-NEXT: v_mov_b32_e32 v3, s15
-; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
+; SDAG-NEXT: v_mov_b32_e32 v2, s12
+; SDAG-NEXT: v_mov_b32_e32 v3, s13
+; SDAG-NEXT: v_mov_b32_e32 v4, s14
+; SDAG-NEXT: v_mov_b32_e32 v5, s15
+; SDAG-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
@@ -3383,28 +3371,28 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b32_e32 v5, s25
; HEURRC-NEXT: v_mov_b32_e32 v6, s26
; HEURRC-NEXT: v_mov_b32_e32 v7, s27
-; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_mov_b32_e32 v8, s28
; HEURRC-NEXT: v_mov_b32_e32 v9, s29
; HEURRC-NEXT: v_mov_b32_e32 v10, s30
; HEURRC-NEXT: v_mov_b32_e32 v11, s31
-; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
-; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
-; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
-; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
-; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
-; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
-; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
-; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
-; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
-; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
-; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
-; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
-; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
-; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
-; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
+; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
+; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
+; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
+; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
+; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
+; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
+; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
+; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
+; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
+; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
+; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
+; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
+; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
+; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: s_nop 1
-; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[4:7], v[8:11], a[0:15] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
@@ -3412,13 +3400,13 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
-; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
-; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
+; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
@@ -3454,60 +3442,56 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
-; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
-; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
-; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
-; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
-; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
-; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
-; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
-; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
+; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
+; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
+; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
+; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
+; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
+; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
-; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[36:39], v[40:43], v[0:15] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: s_nop 11
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
-; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], 16
+; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], 0
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s16
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s17
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s18
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s19
+; VGPRRC-NEXT: global_store_dwordx4 v[0:1], v[20:23], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 0
-; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[2:3], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
-; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
-; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s20
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s21
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s22
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s23
+; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s8
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s9
+; VGPRRC-NEXT: v_mov_b32_e32 v6, s10
+; VGPRRC-NEXT: v_mov_b32_e32 v7, s11
+; VGPRRC-NEXT: global_store_dwordx4 v[2:3], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
-; VGPRRC-NEXT: s_nop 0
-; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
-; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
-; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
-; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
+; VGPRRC-NEXT: v_mov_b32_e32 v2, s12
+; VGPRRC-NEXT: v_mov_b32_e32 v3, s13
+; VGPRRC-NEXT: v_mov_b32_e32 v4, s14
+; VGPRRC-NEXT: v_mov_b32_e32 v5, s15
+; VGPRRC-NEXT: global_store_dwordx4 v[0:1], v[2:5], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 8152b5e38477c..a00f754e726c6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -7494,53 +7494,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; NOLIT-SRCC-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; NOLIT-SRCC-NEXT: v_lshlrev_b32_e32 v24, 7, v0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v26, 1.0
+; NOLIT-SRCC-NEXT: v_mov_b32_e32 v25, 2.0
; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; NOLIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(0)
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1]
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:16
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:32
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:48
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:64
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:80
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:96
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v7
+; NOLIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:112
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v8
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v9
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v10
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v11
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(4)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v12
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v13
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v14
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v15
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(3)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a16, v16
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a17, v17
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v18
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v19
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(2)
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v20
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v21
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a22, v22
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a23, v23
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v24
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v25
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v26
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v27
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v28
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v29
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v30
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v31
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
-; NOLIT-SRCC-NEXT: s_nop 1
-; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(1)
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v1
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v2
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v3
+; NOLIT-SRCC-NEXT: s_waitcnt vmcnt(0)
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v4
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v5
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v6
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v7
+; NOLIT-SRCC-NEXT: s_nop 0
+; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v26, v25, a[0:31] cbsz:1 abid:2 blgp:3
; NOLIT-SRCC-NEXT: s_nop 15
; NOLIT-SRCC-NEXT: s_nop 1
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
@@ -7567,23 +7574,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; NOLIT-SRCC-NEXT: s_nop 0
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1] offset:96
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:112
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[8:11], s[0:1] offset:64
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[12:15], s[0:1] offset:80
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[16:19], s[0:1] offset:32
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[20:23], s[0:1] offset:48
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1]
+; NOLIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:16
; NOLIT-SRCC-NEXT: s_endpgm
;
; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg:
@@ -7684,53 +7690,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_vecarg:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; LIT-SRCC-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; LIT-SRCC-NEXT: v_lshlrev_b32_e32 v24, 7, v0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v26, 1.0
+; LIT-SRCC-NEXT: v_mov_b32_e32 v25, 2.0
; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0)
-; LIT-SRCC-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; LIT-SRCC-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; LIT-SRCC-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; LIT-SRCC-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; LIT-SRCC-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; LIT-SRCC-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; LIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; LIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; LIT-SRCC-NEXT: s_waitcnt vmcnt(0)
+; LIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1]
+; LIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:16
+; LIT-SRCC-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:32
+; LIT-SRCC-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:48
+; LIT-SRCC-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:64
+; LIT-SRCC-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:80
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3
+; LIT-SRCC-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:96
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v7
+; LIT-SRCC-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:112
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(5)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a8, v8
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a9, v9
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a10, v10
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a11, v11
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(4)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a12, v12
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a13, v13
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a14, v14
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a15, v15
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(3)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a16, v16
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a17, v17
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a18, v18
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a19, v19
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(2)
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a20, v20
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a21, v21
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a22, v22
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a23, v23
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v24
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v25
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v26
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v27
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v28
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v29
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v30
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v31
-; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0
-; LIT-SRCC-NEXT: s_nop 1
-; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(1)
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a24, v0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a25, v1
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v2
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v3
+; LIT-SRCC-NEXT: s_waitcnt vmcnt(0)
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v4
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v5
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v6
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v7
+; LIT-SRCC-NEXT: s_nop 0
+; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v26, v25, a[0:31] cbsz:1 abid:2 blgp:3
; LIT-SRCC-NEXT: s_nop 15
; LIT-SRCC-NEXT: s_nop 1
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27
@@ -7757,23 +7770,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13
; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; LIT-SRCC-NEXT: s_nop 0
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5
-; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1] offset:96
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:112
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5
+; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[8:11], s[0:1] offset:64
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[12:15], s[0:1] offset:80
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[16:19], s[0:1] offset:32
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[20:23], s[0:1] offset:48
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1]
+; LIT-SRCC-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:16
; LIT-SRCC-NEXT: s_endpgm
;
; GFX90A-LABEL: test_mfma_f32_32x32x1f32_vecarg:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index e245feb68f587..cbbdf528a947d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji < %s | FileCheck --check-prefix=CHECK-SDAG -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -global-isel -new-reg-bank-select -global-isel-abort=2 < %s | FileCheck --check-prefix=CHECK-GISEL -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -global-isel -global-isel-abort=2 < %s | FileCheck --check-prefix=CHECK-GISEL -enable-var-scope %s
+; NOTE: -new-reg-bank-select fails for i64/f64 readlane due to missing RegBankLegalize rules
declare i32 @llvm.amdgcn.readlane.i32(i32, i32) #0
declare i64 @llvm.amdgcn.readlane.i64(i64, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 996c76488080f..fbf444476bd4c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -870,66 +870,66 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; GCN-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; GCN-NEXT: v_and_b32_e32 v1, 0x1ff80, v0
; GCN-NEXT: v_mov_b32_e32 v2, 1.0
; GCN-NEXT: v_mov_b32_e32 v3, 2.0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s0, v0
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:112
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:96
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:80
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:64
-; GCN-NEXT: ds_read_b128 a[0:3], v1
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:16
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:32
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; GCN-NEXT: v_add_u32_e32 v0, s0, v1
+; GCN-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; GCN-NEXT: ds_read_b128 a[24:27], v0 offset:96
+; GCN-NEXT: ds_read_b128 a[20:23], v0 offset:80
+; GCN-NEXT: ds_read_b128 a[16:19], v0 offset:64
+; GCN-NEXT: ds_read_b128 a[0:3], v0
+; GCN-NEXT: ds_read_b128 a[4:7], v0 offset:16
+; GCN-NEXT: ds_read_b128 a[8:11], v0 offset:32
+; GCN-NEXT: ds_read_b128 a[12:15], v0 offset:48
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_add_u32_e32 v0, s1, v0
+; GCN-NEXT: v_add_u32_e32 v1, s1, v1
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: s_nop 15
; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:112
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:96
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:80
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:64
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:48
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:16
-; GCN-NEXT: ds_write_b128 v0, a[0:3]
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; GCN-NEXT: ds_write_b128 v1, a[28:31] offset:112
+; GCN-NEXT: ds_write_b128 v1, a[24:27] offset:96
+; GCN-NEXT: ds_write_b128 v1, a[20:23] offset:80
+; GCN-NEXT: ds_write_b128 v1, a[16:19] offset:64
+; GCN-NEXT: ds_write_b128 v1, a[12:15] offset:48
+; GCN-NEXT: ds_write_b128 v1, a[8:11] offset:32
+; GCN-NEXT: ds_write_b128 v1, a[4:7] offset:16
+; GCN-NEXT: ds_write_b128 v1, a[0:3]
+; GCN-NEXT: ds_read_b128 a[28:31], v0 offset:8304
+; GCN-NEXT: ds_read_b128 a[24:27], v0 offset:8288
+; GCN-NEXT: ds_read_b128 a[20:23], v0 offset:8272
+; GCN-NEXT: ds_read_b128 a[16:19], v0 offset:8256
+; GCN-NEXT: ds_read_b128 a[12:15], v0 offset:8240
+; GCN-NEXT: ds_read_b128 a[8:11], v0 offset:8224
+; GCN-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; GCN-NEXT: ds_read_b128 a[0:3], v0 offset:8192
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: s_nop 15
; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:24688
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:24672
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:24656
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:24640
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:24624
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:24608
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:24592
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; GCN-NEXT: ds_write_b128 v1, a[24:27] offset:8288
+; GCN-NEXT: ds_write_b128 v1, a[28:31] offset:8304
+; GCN-NEXT: ds_write_b128 v1, a[16:19] offset:8256
+; GCN-NEXT: ds_write_b128 v1, a[20:23] offset:8272
+; GCN-NEXT: ds_write_b128 v1, a[8:11] offset:8224
+; GCN-NEXT: ds_write_b128 v1, a[12:15] offset:8240
+; GCN-NEXT: ds_write_b128 v1, a[0:3] offset:8192
+; GCN-NEXT: ds_write_b128 v1, a[4:7] offset:8208
+; GCN-NEXT: ds_read_b128 a[28:31], v0 offset:24688
+; GCN-NEXT: ds_read_b128 a[24:27], v0 offset:24672
+; GCN-NEXT: ds_read_b128 a[20:23], v0 offset:24656
+; GCN-NEXT: ds_read_b128 a[16:19], v0 offset:24640
+; GCN-NEXT: ds_read_b128 a[12:15], v0 offset:24624
+; GCN-NEXT: ds_read_b128 a[8:11], v0 offset:24608
+; GCN-NEXT: ds_read_b128 a[4:7], v0 offset:24592
+; GCN-NEXT: ds_read_b128 a[0:3], v0 offset:24576
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -937,46 +937,46 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: s_nop 15
; GCN-NEXT: s_nop 2
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:16480
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:16496
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:16448
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:16464
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:16416
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:16432
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:16384
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:16400
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:49264
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:49248
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:49232
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:49216
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:49200
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:49184
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:49168
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; GCN-NEXT: ds_write_b128 v1, a[24:27] offset:16480
+; GCN-NEXT: ds_write_b128 v1, a[28:31] offset:16496
+; GCN-NEXT: ds_write_b128 v1, a[16:19] offset:16448
+; GCN-NEXT: ds_write_b128 v1, a[20:23] offset:16464
+; GCN-NEXT: ds_write_b128 v1, a[8:11] offset:16416
+; GCN-NEXT: ds_write_b128 v1, a[12:15] offset:16432
+; GCN-NEXT: ds_write_b128 v1, a[0:3] offset:16384
+; GCN-NEXT: ds_write_b128 v1, a[4:7] offset:16400
+; GCN-NEXT: ds_read_b128 a[28:31], v0 offset:49264
+; GCN-NEXT: ds_read_b128 a[24:27], v0 offset:49248
+; GCN-NEXT: ds_read_b128 a[20:23], v0 offset:49232
+; GCN-NEXT: ds_read_b128 a[16:19], v0 offset:49216
+; GCN-NEXT: ds_read_b128 a[12:15], v0 offset:49200
+; GCN-NEXT: ds_read_b128 a[8:11], v0 offset:49184
+; GCN-NEXT: ds_read_b128 a[4:7], v0 offset:49168
+; GCN-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; GCN-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; GCN-NEXT: v_add_u32_e32 v0, 0x6000, v0
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: s_nop 15
; GCN-NEXT: s_nop 1
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:24672
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:24688
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:24640
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:24656
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:24608
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:24624
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:24576
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; GCN-NEXT: ds_read_b128 a[28:31], v1 offset:57456
-; GCN-NEXT: ds_read_b128 a[24:27], v1 offset:57440
-; GCN-NEXT: ds_read_b128 a[20:23], v1 offset:57424
-; GCN-NEXT: ds_read_b128 a[16:19], v1 offset:57408
-; GCN-NEXT: ds_read_b128 a[0:3], v1 offset:57344
-; GCN-NEXT: ds_read_b128 a[4:7], v1 offset:57360
-; GCN-NEXT: ds_read_b128 a[8:11], v1 offset:57376
-; GCN-NEXT: ds_read_b128 a[12:15], v1 offset:57392
+; GCN-NEXT: ds_write_b128 v1, a[24:27] offset:24672
+; GCN-NEXT: ds_write_b128 v1, a[28:31] offset:24688
+; GCN-NEXT: ds_write_b128 v1, a[16:19] offset:24640
+; GCN-NEXT: ds_write_b128 v1, a[20:23] offset:24656
+; GCN-NEXT: ds_write_b128 v1, a[8:11] offset:24608
+; GCN-NEXT: ds_write_b128 v1, a[12:15] offset:24624
+; GCN-NEXT: ds_write_b128 v1, a[0:3] offset:24576
+; GCN-NEXT: ds_write_b128 v1, a[4:7] offset:24592
+; GCN-NEXT: ds_read_b128 a[28:31], v0 offset:57456
+; GCN-NEXT: ds_read_b128 a[24:27], v0 offset:57440
+; GCN-NEXT: ds_read_b128 a[20:23], v0 offset:57424
+; GCN-NEXT: ds_read_b128 a[16:19], v0 offset:57408
+; GCN-NEXT: ds_read_b128 a[0:3], v0 offset:57344
+; GCN-NEXT: ds_read_b128 a[4:7], v0 offset:57360
+; GCN-NEXT: ds_read_b128 a[8:11], v0 offset:57376
+; GCN-NEXT: ds_read_b128 a[12:15], v0 offset:57392
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -984,14 +984,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-NEXT: s_nop 15
; GCN-NEXT: s_nop 2
-; GCN-NEXT: ds_write_b128 v0, a[24:27] offset:32864
-; GCN-NEXT: ds_write_b128 v0, a[28:31] offset:32880
-; GCN-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; GCN-NEXT: ds_write_b128 v0, a[20:23] offset:32848
-; GCN-NEXT: ds_write_b128 v0, a[8:11] offset:32800
-; GCN-NEXT: ds_write_b128 v0, a[12:15] offset:32816
-; GCN-NEXT: ds_write_b128 v0, a[0:3] offset:32768
-; GCN-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; GCN-NEXT: ds_write_b128 v1, a[24:27] offset:32864
+; GCN-NEXT: ds_write_b128 v1, a[28:31] offset:32880
+; GCN-NEXT: ds_write_b128 v1, a[16:19] offset:32832
+; GCN-NEXT: ds_write_b128 v1, a[20:23] offset:32848
+; GCN-NEXT: ds_write_b128 v1, a[8:11] offset:32800
+; GCN-NEXT: ds_write_b128 v1, a[12:15] offset:32816
+; GCN-NEXT: ds_write_b128 v1, a[0:3] offset:32768
+; GCN-NEXT: ds_write_b128 v1, a[4:7] offset:32784
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-NEXT: s_endpgm
;
@@ -999,66 +999,66 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; EXACTCUTOFF: ; %bb.0: ; %entry
; EXACTCUTOFF-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 7, v0
-; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x1ff80, v0
+; EXACTCUTOFF-NEXT: v_and_b32_e32 v1, 0x1ff80, v0
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 1.0
; EXACTCUTOFF-NEXT: v_mov_b32_e32 v3, 2.0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s0, v0
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:112
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:96
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:80
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:64
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:16
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:32
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:48
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s0, v1
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v0 offset:112
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v0 offset:96
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v0 offset:80
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v0 offset:64
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v0
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v0 offset:16
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v0 offset:32
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v0 offset:48
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s1, v0
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s1, v1
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_nop 15
; EXACTCUTOFF-NEXT: s_nop 1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3]
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:8240
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[28:31] offset:112
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[24:27] offset:96
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[20:23] offset:80
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[16:19] offset:64
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[12:15] offset:48
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[8:11] offset:32
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[4:7] offset:16
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[0:3]
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v0 offset:8304
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v0 offset:8288
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v0 offset:8272
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v0 offset:8256
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v0 offset:8240
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v0 offset:8224
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v0 offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v0 offset:8192
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s1
+; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, s1
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_nop 15
; EXACTCUTOFF-NEXT: s_nop 1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:24688
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:24672
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:24656
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:24640
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:24624
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:24608
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:24576
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[24:27] offset:8288
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[28:31] offset:8304
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[16:19] offset:8256
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[20:23] offset:8272
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[8:11] offset:8224
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[12:15] offset:8240
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[0:3] offset:8192
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[4:7] offset:8208
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v0 offset:24688
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v0 offset:24672
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v0 offset:24656
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v0 offset:24640
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v0 offset:24624
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v0 offset:24608
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v0 offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v0 offset:24576
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -1066,46 +1066,46 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_nop 15
; EXACTCUTOFF-NEXT: s_nop 2
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:16480
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:16496
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:16448
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:16464
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:16416
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:16432
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:16384
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16400
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:49264
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:49248
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:49232
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:49216
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:49200
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:49184
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:49168
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[24:27] offset:16480
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[28:31] offset:16496
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[16:19] offset:16448
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[20:23] offset:16464
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[8:11] offset:16416
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[12:15] offset:16432
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[0:3] offset:16384
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[4:7] offset:16400
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v0 offset:49264
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v0 offset:49248
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v0 offset:49232
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v0 offset:49216
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v0 offset:49200
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v0 offset:49184
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v0 offset:49168
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
-; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, 0x6000, v1
+; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, 0x6000, v0
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_nop 15
; EXACTCUTOFF-NEXT: s_nop 1
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:24672
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:24688
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:24640
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:24656
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:24608
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592
-; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:57456
-; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:57440
-; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:57424
-; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:57408
-; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:57344
-; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:57360
-; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:57376
-; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:57392
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[24:27] offset:24672
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[28:31] offset:24688
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[16:19] offset:24640
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[20:23] offset:24656
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[8:11] offset:24608
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[12:15] offset:24624
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[0:3] offset:24576
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[4:7] offset:24592
+; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v0 offset:57456
+; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v0 offset:57440
+; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v0 offset:57424
+; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v0 offset:57408
+; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v0 offset:57344
+; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v0 offset:57360
+; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v0 offset:57376
+; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v0 offset:57392
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
@@ -1113,14 +1113,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_nop 15
; EXACTCUTOFF-NEXT: s_nop 2
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:32864
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:32880
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:32832
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:32848
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32800
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:32816
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:32768
-; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:32784
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[24:27] offset:32864
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[28:31] offset:32880
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[16:19] offset:32832
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[20:23] offset:32848
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[8:11] offset:32800
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[12:15] offset:32816
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[0:3] offset:32768
+; EXACTCUTOFF-NEXT: ds_write_b128 v1, a[4:7] offset:32784
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
index 6132baceb1ce3..7e113c850e155 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll
@@ -1700,17 +1700,17 @@ declare <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x64.i8(<4 x i32>, <8 x i32>, <16
define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
; SDAG-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:
; SDAG: ; %bb.0: ; %bb
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-NEXT: v_mov_b32_e32 v24, s8
; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b32_e32 v26, s10
@@ -1719,20 +1719,20 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
-; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:
@@ -1768,17 +1768,17 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
;
; SDAG-VGPR-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:
; SDAG-VGPR: ; %bb.0: ; %bb
-; SDAG-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-VGPR-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
@@ -1787,20 +1787,20 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x64_i8__vgpr(ptr addrspace(1) %a
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s2
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_smfmac_i32_32x32x64_i8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_i32_32x32x64_i8__vgpr:
@@ -3472,17 +3472,17 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.bf8(<4 x i32>, <8 x i3
define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:
; SDAG: ; %bb.0: ; %bb
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-NEXT: v_mov_b32_e32 v24, s8
; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b32_e32 v26, s10
@@ -3491,20 +3491,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
-; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:
@@ -3540,17 +3540,17 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
;
; SDAG-VGPR-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:
; SDAG-VGPR: ; %bb.0: ; %bb
-; SDAG-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-VGPR-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
@@ -3559,20 +3559,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s2
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_32x32x64_bf8_bf8__vgpr:
@@ -4080,17 +4080,17 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.bf8.fp8(<4 x i32>, <8 x i3
define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
; SDAG-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:
; SDAG: ; %bb.0: ; %bb
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-NEXT: v_mov_b32_e32 v24, s8
; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b32_e32 v26, s10
@@ -4099,20 +4099,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
-; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:
@@ -4148,17 +4148,17 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
;
; SDAG-VGPR-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:
; SDAG-VGPR: ; %bb.0: ; %bb
-; SDAG-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-VGPR-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
@@ -4167,20 +4167,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_bf8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s2
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_bf8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_32x32x64_bf8_fp8__vgpr:
@@ -4688,17 +4688,17 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.bf8(<4 x i32>, <8 x i3
define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:
; SDAG: ; %bb.0: ; %bb
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-NEXT: v_mov_b32_e32 v24, s8
; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b32_e32 v26, s10
@@ -4707,20 +4707,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
-; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:
@@ -4756,17 +4756,17 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
;
; SDAG-VGPR-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:
; SDAG-VGPR: ; %bb.0: ; %bb
-; SDAG-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-VGPR-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
@@ -4775,20 +4775,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_bf8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s2
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_bf8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_32x32x64_fp8_bf8__vgpr:
@@ -5296,17 +5296,17 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x64.fp8.fp8(<4 x i32>, <8 x i3
define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
; SDAG-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:
; SDAG: ; %bb.0: ; %bb
-; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-NEXT: v_mov_b32_e32 v24, s8
; SDAG-NEXT: v_mov_b32_e32 v25, s9
; SDAG-NEXT: v_mov_b32_e32 v26, s10
@@ -5315,20 +5315,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-NEXT: v_mov_b32_e32 v28, s2
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 10
-; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:
@@ -5364,17 +5364,17 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
;
; SDAG-VGPR-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:
; SDAG-VGPR: ; %bb.0: ; %bb
-; SDAG-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; SDAG-VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; SDAG-VGPR-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; SDAG-VGPR-NEXT: v_lshlrev_b32_e32 v16, 6, v0
; SDAG-VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
-; SDAG-VGPR-NEXT: s_load_dword s16, s[4:5], 0x64
-; SDAG-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
+; SDAG-VGPR-NEXT: s_load_dword s2, s[4:5], 0x64
+; SDAG-VGPR-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x54
; SDAG-VGPR-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[6:7] offset:16
-; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7]
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
+; SDAG-VGPR-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
; SDAG-VGPR-NEXT: v_mov_b32_e32 v24, s8
; SDAG-VGPR-NEXT: v_mov_b32_e32 v25, s9
; SDAG-VGPR-NEXT: v_mov_b32_e32 v26, s10
@@ -5383,20 +5383,20 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x64_fp8_fp8__vgpr(ptr addrspace(
; SDAG-VGPR-NEXT: v_mov_b32_e32 v17, s13
; SDAG-VGPR-NEXT: v_mov_b32_e32 v18, s14
; SDAG-VGPR-NEXT: v_mov_b32_e32 v19, s15
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s0
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s1
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s2
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s3
-; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v20, s16
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v21, s17
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v22, s18
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v23, s19
+; SDAG-VGPR-NEXT: v_mov_b32_e32 v28, s2
; SDAG-VGPR-NEXT: s_waitcnt vmcnt(0)
; SDAG-VGPR-NEXT: s_nop 0
; SDAG-VGPR-NEXT: v_smfmac_f32_32x32x64_fp8_fp8 v[0:15], v[24:27], v[16:23], v28 cbsz:1 abid:2
; SDAG-VGPR-NEXT: v_mov_b32_e32 v16, 0
; SDAG-VGPR-NEXT: s_nop 10
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
-; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
+; SDAG-VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-VGPR-NEXT: s_endpgm
;
; GISEL-VGPR-LABEL: test_smfmac_f32_32x32x64_fp8_fp8__vgpr:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index 76b97e843d777..71099f97c91c4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -76,11 +76,12 @@ define amdgpu_kernel void @v_round_f64(ptr addrspace(1) %out, ptr addrspace(1) %
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_movk_i32 s4, 0xfc01
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_mov_b32 s3, 0xfffff
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_bfe_u32 v4, v3, 20, 11
-; SI-NEXT: v_add_i32_e32 v6, vcc, 0xfffffc01, v4
+; SI-NEXT: v_add_i32_e32 v6, vcc, s4, v4
; SI-NEXT: v_lshr_b64 v[4:5], s[2:3], v6
; SI-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; SI-NEXT: v_bfi_b32 v5, v5, 0, v3
@@ -570,87 +571,87 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x19
; CI-NEXT: s_brev_b32 s6, -2
-; CI-NEXT: v_mov_b32_e32 v12, 0
+; CI-NEXT: v_mov_b32_e32 v4, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_trunc_f64_e32 v[0:1], s[10:11]
-; CI-NEXT: v_trunc_f64_e32 v[4:5], s[8:9]
+; CI-NEXT: v_trunc_f64_e32 v[6:7], s[8:9]
; CI-NEXT: v_add_f64 v[2:3], s[10:11], -v[0:1]
-; CI-NEXT: v_add_f64 v[6:7], s[8:9], -v[4:5]
+; CI-NEXT: v_add_f64 v[8:9], s[8:9], -v[6:7]
; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[2:3]|, 0.5
-; CI-NEXT: v_cmp_ge_f64_e64 s[2:3], |v[6:7]|, 0.5
+; CI-NEXT: v_cmp_ge_f64_e64 s[2:3], |v[8:9]|, 0.5
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; CI-NEXT: s_cselect_b32 s7, 0x3ff00000, 0
-; CI-NEXT: v_mov_b32_e32 v8, s11
+; CI-NEXT: v_mov_b32_e32 v5, s11
; CI-NEXT: s_and_b64 s[0:1], s[2:3], exec
; CI-NEXT: v_mov_b32_e32 v2, s7
-; CI-NEXT: v_trunc_f64_e32 v[6:7], s[14:15]
-; CI-NEXT: v_bfi_b32 v13, s6, v2, v8
+; CI-NEXT: v_trunc_f64_e32 v[8:9], s[14:15]
+; CI-NEXT: v_bfi_b32 v5, s6, v2, v5
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_add_f64 v[2:3], v[0:1], v[12:13]
-; CI-NEXT: v_mov_b32_e32 v8, s0
-; CI-NEXT: v_mov_b32_e32 v9, s9
-; CI-NEXT: v_add_f64 v[0:1], s[14:15], -v[6:7]
-; CI-NEXT: v_bfi_b32 v13, s6, v8, v9
+; CI-NEXT: v_add_f64 v[2:3], v[0:1], v[4:5]
+; CI-NEXT: v_mov_b32_e32 v5, s0
+; CI-NEXT: v_mov_b32_e32 v10, s9
+; CI-NEXT: v_add_f64 v[0:1], s[14:15], -v[8:9]
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v10
; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[0:1]|, 0.5
-; CI-NEXT: v_add_f64 v[0:1], v[4:5], v[12:13]
-; CI-NEXT: v_trunc_f64_e32 v[4:5], s[12:13]
+; CI-NEXT: v_add_f64 v[0:1], v[6:7], v[4:5]
+; CI-NEXT: v_trunc_f64_e32 v[6:7], s[12:13]
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: v_add_f64 v[8:9], s[12:13], -v[4:5]
+; CI-NEXT: v_add_f64 v[10:11], s[12:13], -v[6:7]
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_mov_b32_e32 v10, s0
-; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[8:9]|, 0.5
-; CI-NEXT: v_trunc_f64_e32 v[8:9], s[18:19]
-; CI-NEXT: v_mov_b32_e32 v11, s15
-; CI-NEXT: v_bfi_b32 v13, s6, v10, v11
+; CI-NEXT: v_mov_b32_e32 v5, s0
+; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[10:11]|, 0.5
+; CI-NEXT: v_trunc_f64_e32 v[10:11], s[18:19]
+; CI-NEXT: v_mov_b32_e32 v12, s15
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: v_add_f64 v[10:11], s[18:19], -v[8:9]
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v12
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_add_f64 v[6:7], v[6:7], v[12:13]
-; CI-NEXT: v_mov_b32_e32 v13, s0
+; CI-NEXT: v_add_f64 v[12:13], s[18:19], -v[10:11]
+; CI-NEXT: v_add_f64 v[8:9], v[8:9], v[4:5]
+; CI-NEXT: v_mov_b32_e32 v5, s0
; CI-NEXT: v_mov_b32_e32 v14, s13
-; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[10:11]|, 0.5
-; CI-NEXT: v_bfi_b32 v13, s6, v13, v14
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v14
+; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[12:13]|, 0.5
; CI-NEXT: v_trunc_f64_e32 v[14:15], s[16:17]
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CI-NEXT: v_add_f64 v[12:13], s[16:17], -v[14:15]
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_add_f64 v[10:11], s[16:17], -v[14:15]
-; CI-NEXT: v_add_f64 v[4:5], v[4:5], v[12:13]
-; CI-NEXT: v_mov_b32_e32 v13, s0
+; CI-NEXT: v_add_f64 v[6:7], v[6:7], v[4:5]
+; CI-NEXT: v_mov_b32_e32 v5, s0
+; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[12:13]|, 0.5
; CI-NEXT: v_mov_b32_e32 v16, s19
-; CI-NEXT: v_bfi_b32 v13, s6, v13, v16
-; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[10:11]|, 0.5
-; CI-NEXT: v_trunc_f64_e32 v[16:17], s[22:23]
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: v_add_f64 v[18:19], s[22:23], -v[16:17]
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v16
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_add_f64 v[10:11], v[8:9], v[12:13]
-; CI-NEXT: v_mov_b32_e32 v8, s0
-; CI-NEXT: v_mov_b32_e32 v9, s17
+; CI-NEXT: v_trunc_f64_e32 v[16:17], s[22:23]
+; CI-NEXT: v_add_f64 v[12:13], v[10:11], v[4:5]
+; CI-NEXT: v_mov_b32_e32 v5, s0
+; CI-NEXT: v_mov_b32_e32 v10, s17
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v10
+; CI-NEXT: v_add_f64 v[18:19], s[22:23], -v[16:17]
+; CI-NEXT: v_add_f64 v[10:11], v[14:15], v[4:5]
+; CI-NEXT: v_trunc_f64_e32 v[14:15], s[20:21]
; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[18:19]|, 0.5
-; CI-NEXT: v_trunc_f64_e32 v[18:19], s[20:21]
-; CI-NEXT: v_bfi_b32 v13, s6, v8, v9
-; CI-NEXT: v_add_f64 v[8:9], v[14:15], v[12:13]
-; CI-NEXT: v_add_f64 v[13:14], s[20:21], -v[18:19]
+; CI-NEXT: v_add_f64 v[18:19], s[20:21], -v[14:15]
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[13:14]|, 0.5
+; CI-NEXT: v_cmp_ge_f64_e64 s[0:1], |v[18:19]|, 0.5
; CI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CI-NEXT: v_mov_b32_e32 v5, s2
+; CI-NEXT: v_mov_b32_e32 v18, s23
; CI-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; CI-NEXT: v_mov_b32_e32 v13, s2
-; CI-NEXT: v_mov_b32_e32 v14, s23
-; CI-NEXT: v_mov_b32_e32 v20, s0
+; CI-NEXT: v_bfi_b32 v5, s6, v5, v18
+; CI-NEXT: v_mov_b32_e32 v18, s0
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: v_bfi_b32 v13, s6, v13, v14
-; CI-NEXT: v_mov_b32_e32 v21, s21
-; CI-NEXT: v_add_f64 v[14:15], v[16:17], v[12:13]
-; CI-NEXT: v_bfi_b32 v13, s6, v20, v21
-; CI-NEXT: v_add_f64 v[12:13], v[18:19], v[12:13]
+; CI-NEXT: v_mov_b32_e32 v19, s21
+; CI-NEXT: v_add_f64 v[16:17], v[16:17], v[4:5]
+; CI-NEXT: v_bfi_b32 v5, s6, v18, v19
+; CI-NEXT: v_add_f64 v[14:15], v[14:15], v[4:5]
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
-; CI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
-; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; CI-NEXT: buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:48
+; CI-NEXT: buffer_store_dwordx4 v[10:13], off, s[0:3], 0 offset:32
+; CI-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:16
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; CI-NEXT: s_endpgm
%result = call <8 x double> @llvm.round.v8f64(<8 x double> %in) #1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
index 5cbe3e72ce5f9..fffd2c7073453 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
@@ -1625,8 +1625,9 @@ define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
; GFX6-NEXT: s_add_i32 s1, s0, 4
; GFX6-NEXT: s_cmp_lt_u32 s0, 4
; GFX6-NEXT: s_cselect_b32 s4, s0, s1
-; GFX6-NEXT: s_mov_b64 s[0:1], 0
+; GFX6-NEXT: s_mov_b32 s0, 0
; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_mov_b32 s1, s0
; GFX6-NEXT: v_mov_b32_e32 v0, s4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -1645,8 +1646,9 @@ define amdgpu_kernel void @get_rounding_after_set_rounding_1() {
; GFX7-NEXT: s_add_i32 s1, s0, 4
; GFX7-NEXT: s_cmp_lt_u32 s0, 4
; GFX7-NEXT: s_cselect_b32 s4, s0, s1
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: s_mov_b32 s0, 0
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s1, s0
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 9d18dd541d15e..997090e044fee 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -7544,65 +7544,65 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
;
; GFX12-LABEL: constant_sextload_v16i16_to_v16i64:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_load_b256 s[4:11], s[2:3], 0x0
+; GFX12-NEXT: s_load_b256 s[0:7], s[10:11], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s30, s9
-; GFX12-NEXT: s_lshr_b32 s34, s9, 16
-; GFX12-NEXT: s_bfe_i64 s[28:29], s[8:9], 0x100000
-; GFX12-NEXT: s_lshr_b32 s8, s8, 16
-; GFX12-NEXT: s_bfe_i64 s[22:23], s[10:11], 0x100000
-; GFX12-NEXT: s_mov_b32 s24, s11
-; GFX12-NEXT: s_lshr_b32 s26, s11, 16
+; GFX12-NEXT: s_mov_b32 s30, s5
+; GFX12-NEXT: s_lshr_b32 s34, s5, 16
+; GFX12-NEXT: s_bfe_i64 s[28:29], s[4:5], 0x100000
+; GFX12-NEXT: s_lshr_b32 s4, s4, 16
+; GFX12-NEXT: s_bfe_i64 s[22:23], s[6:7], 0x100000
+; GFX12-NEXT: s_mov_b32 s24, s7
+; GFX12-NEXT: s_lshr_b32 s26, s7, 16
; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x100000
; GFX12-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x100000
-; GFX12-NEXT: s_lshr_b32 s10, s10, 16
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x100000
+; GFX12-NEXT: s_lshr_b32 s6, s6, 16
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s29
-; GFX12-NEXT: s_mov_b32 s18, s7
-; GFX12-NEXT: s_lshr_b32 s20, s7, 16
+; GFX12-NEXT: s_mov_b32 s18, s3
+; GFX12-NEXT: s_lshr_b32 s20, s3, 16
; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x100000
; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v4, s22 :: v_dual_mov_b32 v9, s31
; GFX12-NEXT: v_dual_mov_b32 v8, s30 :: v_dual_mov_b32 v11, s35
-; GFX12-NEXT: v_dual_mov_b32 v10, s34 :: v_dual_mov_b32 v3, s9
-; GFX12-NEXT: s_bfe_i64 s[16:17], s[6:7], 0x100000
-; GFX12-NEXT: s_lshr_b32 s6, s6, 16
-; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v10, s34 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x100000
+; GFX12-NEXT: s_lshr_b32 s2, s2, 16
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v0, s28 :: v_dual_mov_b32 v5, s23
-; GFX12-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v13, s25
-; GFX12-NEXT: s_mov_b32 s12, s5
-; GFX12-NEXT: s_lshr_b32 s14, s5, 16
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v13, s25
+; GFX12-NEXT: s_mov_b32 s12, s1
+; GFX12-NEXT: s_lshr_b32 s14, s1, 16
; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x100000
; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v15, s27
-; GFX12-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v7, s11
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[4:5], 0x100000
-; GFX12-NEXT: s_lshr_b32 s4, s4, 16
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x100000
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v17, s19
+; GFX12-NEXT: v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: s_bfe_i64 s[10:11], s[0:1], 0x100000
+; GFX12-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v17, s19
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x100000
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
; GFX12-NEXT: v_mov_b32_e32 v18, s20
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x100000
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
-; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:64
-; GFX12-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s7
-; GFX12-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v2, s6
+; GFX12-NEXT: global_store_b128 v24, v[8:11], s[8:9] offset:80
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:64
+; GFX12-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v2, s2
; GFX12-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v8, s12
; GFX12-NEXT: v_dual_mov_b32 v11, s15 :: v_dual_mov_b32 v10, s14
-; GFX12-NEXT: v_dual_mov_b32 v21, s3 :: v_dual_mov_b32 v20, s2
-; GFX12-NEXT: v_dual_mov_b32 v23, s5 :: v_dual_mov_b32 v22, s4
+; GFX12-NEXT: v_dual_mov_b32 v21, s11 :: v_dual_mov_b32 v20, s10
+; GFX12-NEXT: v_dual_mov_b32 v23, s1 :: v_dual_mov_b32 v22, s0
; GFX12-NEXT: s_clause 0x5
-; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:112
-; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
-; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:48
-; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX12-NEXT: global_store_b128 v24, v[12:15], s[8:9] offset:112
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:96
+; GFX12-NEXT: global_store_b128 v24, v[16:19], s[8:9] offset:48
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:32
+; GFX12-NEXT: global_store_b128 v24, v[8:11], s[8:9] offset:16
+; GFX12-NEXT: global_store_b128 v24, v[20:23], s[8:9]
; GFX12-NEXT: s_endpgm
%load = load <16 x i16>, ptr addrspace(4) %in
%ext = sext <16 x i16> %load to <16 x i64>
@@ -9056,7 +9056,7 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_lshr_b32 s28, s2, 16
; GFX12-NEXT: s_lshr_b32 s42, s5, 16
-; GFX12-NEXT: s_lshr_b32 s50, s8, 16
+; GFX12-NEXT: s_lshr_b32 s52, s8, 16
; GFX12-NEXT: s_mov_b32 s60, s11
; GFX12-NEXT: s_lshr_b32 s22, s0, 16
; GFX12-NEXT: s_mov_b32 s24, s1
@@ -9065,18 +9065,18 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: s_lshr_b32 s36, s3, 16
; GFX12-NEXT: s_lshr_b32 s38, s4, 16
; GFX12-NEXT: s_mov_b32 s40, s5
-; GFX12-NEXT: s_lshr_b32 s44, s6, 16
-; GFX12-NEXT: s_mov_b32 s46, s7
-; GFX12-NEXT: s_lshr_b32 s48, s7, 16
-; GFX12-NEXT: s_mov_b32 s52, s9
-; GFX12-NEXT: s_lshr_b32 s54, s9, 16
-; GFX12-NEXT: s_bfe_i64 s[56:57], s[10:11], 0x100000
+; GFX12-NEXT: s_lshr_b32 s46, s6, 16
+; GFX12-NEXT: s_mov_b32 s48, s7
+; GFX12-NEXT: s_lshr_b32 s50, s7, 16
+; GFX12-NEXT: s_mov_b32 s54, s9
+; GFX12-NEXT: s_lshr_b32 s56, s9, 16
+; GFX12-NEXT: s_bfe_i64 s[44:45], s[10:11], 0x100000
; GFX12-NEXT: s_lshr_b32 s58, s10, 16
; GFX12-NEXT: s_lshr_b32 s62, s11, 16
; GFX12-NEXT: s_bfe_i64 s[10:11], s[28:29], 0x100000
; GFX12-NEXT: s_bfe_i64 s[28:29], s[42:43], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[42:43], s[50:51], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[50:51], s[60:61], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[42:43], s[52:53], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[52:53], s[60:61], 0x100000
; GFX12-NEXT: s_lshr_b32 s60, s14, 16
; GFX12-NEXT: s_bfe_i64 s[64:65], s[14:15], 0x100000
; GFX12-NEXT: s_mov_b32 s14, s15
@@ -9093,14 +9093,14 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: s_bfe_i64 s[22:23], s[36:37], 0x100000
; GFX12-NEXT: s_bfe_i64 s[26:27], s[38:39], 0x100000
; GFX12-NEXT: s_bfe_i64 s[30:31], s[40:41], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[36:37], s[44:45], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[40:41], s[46:47], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[38:39], s[48:49], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[44:45], s[52:53], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[36:37], s[46:47], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[40:41], s[48:49], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[38:39], s[50:51], 0x100000
; GFX12-NEXT: s_bfe_i64 s[46:47], s[54:55], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[48:49], s[58:59], 0x100000
-; GFX12-NEXT: s_lshr_b32 s52, s12, 16
-; GFX12-NEXT: s_bfe_i64 s[54:55], s[12:13], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[48:49], s[56:57], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[50:51], s[58:59], 0x100000
+; GFX12-NEXT: s_lshr_b32 s54, s12, 16
+; GFX12-NEXT: s_bfe_i64 s[56:57], s[12:13], 0x100000
; GFX12-NEXT: s_mov_b32 s12, s13
; GFX12-NEXT: s_lshr_b32 s58, s13, 16
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x100000
@@ -9111,13 +9111,13 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: s_bfe_i64 s[58:59], s[58:59], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v0, s14 :: v_dual_mov_b32 v3, s67
; GFX12-NEXT: v_dual_mov_b32 v2, s66 :: v_dual_mov_b32 v5, s65
-; GFX12-NEXT: s_bfe_i64 s[52:53], s[52:53], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[54:55], s[54:55], 0x100000
; GFX12-NEXT: v_dual_mov_b32 v4, s64 :: v_dual_mov_b32 v7, s61
; GFX12-NEXT: v_dual_mov_b32 v6, s60 :: v_dual_mov_b32 v9, s13
; GFX12-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s59
-; GFX12-NEXT: v_dual_mov_b32 v10, s58 :: v_dual_mov_b32 v13, s55
-; GFX12-NEXT: v_dual_mov_b32 v12, s54 :: v_dual_mov_b32 v15, s53
-; GFX12-NEXT: v_mov_b32_e32 v14, s52
+; GFX12-NEXT: v_dual_mov_b32 v10, s58 :: v_dual_mov_b32 v13, s57
+; GFX12-NEXT: v_dual_mov_b32 v12, s56 :: v_dual_mov_b32 v15, s55
+; GFX12-NEXT: v_mov_b32_e32 v14, s54
; GFX12-NEXT: s_bfe_i64 s[12:13], s[62:63], 0x100000
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[16:17] offset:240
@@ -9125,12 +9125,12 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; GFX12-NEXT: global_store_b128 v24, v[8:11], s[16:17] offset:208
; GFX12-NEXT: global_store_b128 v24, v[12:15], s[16:17] offset:192
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s50 :: v_dual_mov_b32 v3, s13
-; GFX12-NEXT: v_dual_mov_b32 v1, s51 :: v_dual_mov_b32 v2, s12
-; GFX12-NEXT: v_dual_mov_b32 v5, s57 :: v_dual_mov_b32 v4, s56
-; GFX12-NEXT: v_dual_mov_b32 v7, s49 :: v_dual_mov_b32 v6, s48
-; GFX12-NEXT: v_dual_mov_b32 v9, s45 :: v_dual_mov_b32 v8, s44
-; GFX12-NEXT: v_dual_mov_b32 v11, s47 :: v_dual_mov_b32 v10, s46
+; GFX12-NEXT: v_dual_mov_b32 v0, s52 :: v_dual_mov_b32 v3, s13
+; GFX12-NEXT: v_dual_mov_b32 v1, s53 :: v_dual_mov_b32 v2, s12
+; GFX12-NEXT: v_dual_mov_b32 v5, s45 :: v_dual_mov_b32 v4, s44
+; GFX12-NEXT: v_dual_mov_b32 v7, s51 :: v_dual_mov_b32 v6, s50
+; GFX12-NEXT: v_dual_mov_b32 v9, s47 :: v_dual_mov_b32 v8, s46
+; GFX12-NEXT: v_dual_mov_b32 v11, s49 :: v_dual_mov_b32 v10, s48
; GFX12-NEXT: v_dual_mov_b32 v13, s35 :: v_dual_mov_b32 v12, s34
; GFX12-NEXT: v_dual_mov_b32 v15, s43 :: v_dual_mov_b32 v14, s42
; GFX12-NEXT: v_dual_mov_b32 v17, s41 :: v_dual_mov_b32 v16, s40
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index a44fd4b1b64f3..1699e4f3d95ba 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -974,31 +974,31 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
;
; GFX7-HSA-LABEL: constant_load_v11i32:
; GFX7-HSA: ; %bb.0: ; %entry
-; GFX7-HSA-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
+; GFX7-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX7-HSA-NEXT: s_add_i32 s12, s12, s17
; GFX7-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX7-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-HSA-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0
-; GFX7-HSA-NEXT: s_load_dwordx4 s[12:15], s[10:11], 0x8
-; GFX7-HSA-NEXT: s_add_u32 s10, s8, 16
-; GFX7-HSA-NEXT: s_addc_u32 s11, s9, 0
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s10
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s11
+; GFX7-HSA-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x0
+; GFX7-HSA-NEXT: s_load_dwordx4 s[12:15], s[2:3], 0x8
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 16
+; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
+; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s2
; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s6
-; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s7
-; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-HSA-NEXT: flat_store_dwordx4 v[5:6], v[0:3]
-; GFX7-HSA-NEXT: s_add_u32 s0, s8, 32
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-HSA-NEXT: s_addc_u32 s1, s9, 0
+; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s10
+; GFX7-HSA-NEXT: v_mov_b32_e32 v3, s11
+; GFX7-HSA-NEXT: flat_store_dwordx4 v[5:6], v[0:3]
+; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-HSA-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-HSA-NEXT: s_add_u32 s0, s0, 32
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-HSA-NEXT: v_mov_b32_e32 v6, s6
+; GFX7-HSA-NEXT: v_mov_b32_e32 v7, s7
+; GFX7-HSA-NEXT: s_addc_u32 s1, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s12
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s1
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
index c119ef274bb04..44b7b1af0e79b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
@@ -549,25 +549,25 @@ define amdgpu_kernel void @global_load_v11f32(ptr addrspace(1) %out, ptr addrspa
;
; GCNX3-NOHSA-LABEL: global_load_v11f32:
; GCNX3-NOHSA: ; %bb.0: ; %entry
-; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT: s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT: s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s3
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s7
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: buffer_load_dwordx3 v[8:10], off, s[8:11], 0 offset:32
-; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT: s_mov_b32 s1, s5
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx3 v[8:10], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx3 v[8:10], off, s[0:3], 0 offset:32
; GCNX3-NOHSA-NEXT: s_endpgm
entry:
%tmp0 = load <11 x float>, ptr addrspace(1) %in
@@ -639,25 +639,25 @@ define amdgpu_kernel void @global_load_v12f32(ptr addrspace(1) %out, ptr addrspa
;
; GCNX3-NOHSA-LABEL: global_load_v12f32:
; GCNX3-NOHSA: ; %bb.0: ; %entry
-; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT: s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT: s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s3
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s7
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:32
-; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT: s_mov_b32 s1, s5
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
; GCNX3-NOHSA-NEXT: s_endpgm
entry:
%tmp0 = load <12 x float>, ptr addrspace(1) %in
@@ -668,28 +668,28 @@ entry:
define amdgpu_kernel void @global_load_v16f32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; SI-NOHSA-LABEL: global_load_v16f32:
; SI-NOHSA: ; %bb.0: ; %entry
-; SI-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; SI-NOHSA-NEXT: s_mov_b32 s6, -1
-; SI-NOHSA-NEXT: s_mov_b32 s10, s6
-; SI-NOHSA-NEXT: s_mov_b32 s11, s7
+; SI-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; SI-NOHSA-NEXT: s_mov_b32 s2, -1
+; SI-NOHSA-NEXT: s_mov_b32 s10, s2
+; SI-NOHSA-NEXT: s_mov_b32 s11, s3
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NOHSA-NEXT: s_mov_b32 s4, s0
-; SI-NOHSA-NEXT: s_mov_b32 s5, s1
-; SI-NOHSA-NEXT: s_mov_b32 s8, s2
-; SI-NOHSA-NEXT: s_mov_b32 s9, s3
+; SI-NOHSA-NEXT: s_mov_b32 s0, s4
+; SI-NOHSA-NEXT: s_mov_b32 s1, s5
+; SI-NOHSA-NEXT: s_mov_b32 s8, s6
+; SI-NOHSA-NEXT: s_mov_b32 s9, s7
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:32
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:16
; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:32
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:48
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
; SI-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; SI-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:16
+; SI-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
; SI-NOHSA-NEXT: s_endpgm
;
; GCN-HSA-LABEL: global_load_v16f32:
@@ -743,28 +743,28 @@ define amdgpu_kernel void @global_load_v16f32(ptr addrspace(1) %out, ptr addrspa
;
; GCNX3-NOHSA-LABEL: global_load_v16f32:
; GCNX3-NOHSA: ; %bb.0: ; %entry
-; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT: s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT: s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s3
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s7
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:32
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:48
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:16
-; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT: s_mov_b32 s1, s5
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:48
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(3)
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
; GCNX3-NOHSA-NEXT: s_endpgm
entry:
%tmp0 = load <16 x float>, ptr addrspace(1) %in
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
index 0499b007575c8..1ed72b4e23fad 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -2042,36 +2042,36 @@ define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(ptr addrspace(1) %out,
;
; GCNX3-NOHSA-LABEL: global_zextload_v8i32_to_v8i64:
; GCNX3-NOHSA: ; %bb.0:
-; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCNX3-NOHSA-NEXT: s_mov_b32 s7, 0xf000
-; GCNX3-NOHSA-NEXT: s_mov_b32 s6, -1
-; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s6
-; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s7
+; GCNX3-NOHSA-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCNX3-NOHSA-NEXT: s_mov_b32 s3, 0xf000
+; GCNX3-NOHSA-NEXT: s_mov_b32 s2, -1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s10, s2
+; GCNX3-NOHSA-NEXT: s_mov_b32 s11, s3
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
-; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
+; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s6
+; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s7
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v9, 0
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v11, v9
-; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
-; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
+; GCNX3-NOHSA-NEXT: s_mov_b32 s0, s4
+; GCNX3-NOHSA-NEXT: s_mov_b32 s1, s5
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(1)
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v8, v2
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v10, v3
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:48
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
; GCNX3-NOHSA-NEXT: s_nop 0
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v8, v0
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v10, v1
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:32
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(2)
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v8, v6
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v10, v7
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:16
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
; GCNX3-NOHSA-NEXT: s_nop 0
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v8, v4
; GCNX3-NOHSA-NEXT: v_mov_b32_e32 v10, v5
-; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
+; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
; GCNX3-NOHSA-NEXT: s_endpgm
;
; EG-LABEL: global_zextload_v8i32_to_v8i64:
@@ -2431,26 +2431,26 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
; GCNX3-HSA: ; %bb.0:
; GCNX3-HSA-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCNX3-HSA-NEXT: s_add_i32 s12, s12, s17
-; GCNX3-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCNX3-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCNX3-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCNX3-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 48
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 32
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
@@ -2501,31 +2501,30 @@ define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[24:25], v[16:19]
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[26:27], v[8:11]
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(5)
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v15, 31, v5
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v11, 31, v7
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v9, 31, v6
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v13, 31, v4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, v4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, v5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, v6
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, v7
+; GCNX3-HSA-NEXT: s_waitcnt vmcnt(4)
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v15, 31, v1
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v11, 31, v3
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v9, 31, v2
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v13, 31, v0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, v0
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, v1
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, v2
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, v3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v22, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v31, s1
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[20:21], v[12:15]
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[28:29], v[8:11]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v30, s0
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(6)
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v11, 31, v1
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v9, 31, v0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, v0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, v1
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v7, 31, v3
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v5, 31, v2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, v2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, v3
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v11, 31, v5
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v9, 31, v4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, v4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, v5
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v3, 31, v7
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v1, 31, v6
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v6
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v7
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[22:23], v[8:11]
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[30:31], v[4:7]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[30:31], v[0:3]
; GCNX3-HSA-NEXT: s_endpgm
;
; GCNX3-NOHSA-LABEL: global_sextload_v16i32_to_v16i64:
@@ -3210,26 +3209,26 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 64
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s5
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
; GCNX3-HSA-NEXT: s_add_u32 s4, s2, 48
; GCNX3-HSA-NEXT: s_addc_u32 s5, s3, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s4
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[16:19], v[8:9]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s5
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s4
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[16:19], v[12:13]
; GCNX3-HSA-NEXT: s_add_u32 s6, s2, 32
; GCNX3-HSA-NEXT: s_addc_u32 s7, s3, 0
; GCNX3-HSA-NEXT: s_add_u32 s2, s2, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s3, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s7
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s7
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s6
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s6
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v37, s1
@@ -3296,24 +3295,24 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[34:35], v[24:27]
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[36:37], v[28:31]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(10)
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v23, 31, v15
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v21, 31, v14
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v27, 31, v13
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v25, 31, v12
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v24, v12
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v26, v13
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v20, v14
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v22, v15
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v23, 31, v11
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v21, 31, v10
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v27, 31, v9
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v25, 31, v8
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v24, v8
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v26, v9
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v20, v10
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v22, v11
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(9)
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v15, 31, v5
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v13, 31, v4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, v4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v14, v5
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v11, 31, v5
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v9, 31, v4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, v4
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v10, v5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x90
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x60
@@ -3326,58 +3325,58 @@ define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[4:5], v[23:26]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v7, 31, v17
; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v5, 31, v16
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, v16
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, s2
-; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x70
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v7, 31, v17
; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, v17
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v17, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s2
+; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x70
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[15:16], v[4:7]
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, s2
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v17, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 64
; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v26, 31, v19
; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v24, 31, v18
; GCNX3-HSA-NEXT: v_mov_b32_e32 v23, v18
; GCNX3-HSA-NEXT: v_mov_b32_e32 v25, v19
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[15:16], v[23:26]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[16:17], v[23:26]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v18, 31, v9
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v16, 31, v8
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v15, v8
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v17, v9
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v17, 31, v12
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v16, v12
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x50
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v19, 31, v13
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v18, v13
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[15:18]
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[11:12], v[16:19]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s3
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v7, 31, v15
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v5, 31, v14
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, v14
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, v15
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 32
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v14, 31, v1
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v12, 31, v0
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v7, 31, v11
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v6, v11
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v11, v0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, v1
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v10, 31, v1
+; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v8, 31, v0
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[11:12], v[4:7]
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, v1
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v7, v0
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 48
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[0:1], v[11:14]
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[0:1], v[7:10]
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s0
; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v22, 31, v3
; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v20, 31, v2
-; GCNX3-HSA-NEXT: v_ashrrev_i32_e32 v5, 31, v10
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, v10
; GCNX3-HSA-NEXT: v_mov_b32_e32 v19, v2
; GCNX3-HSA-NEXT: v_mov_b32_e32 v21, v3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s1
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[0:1], v[19:22]
; GCNX3-HSA-NEXT: s_endpgm
;
@@ -4016,13 +4015,13 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, s8
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, s9
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[0:1]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v4, s4
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s7
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s7
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s5
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s6
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s6
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, 0
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: v_mov_b32_e32 v3, v1
@@ -4107,29 +4106,29 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v19
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[26:27], v[0:3]
; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v12
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v13
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v13, s3
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v12, s2
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v8
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v9
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 64
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[30:31], v[0:3]
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v14
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v15
-; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v8
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v9
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v10
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v11
+; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 0x50
+; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v12
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v13
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s2
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 32
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v10
-; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v11
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v14
+; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v15
; GCNX3-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 48
diff --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index cf6732d30d080..20f919a97ad14 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1055,9 +1055,9 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-STD-NEXT: s_waitcnt lgkmcnt(0)
; SI-STD-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-STD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-STD-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-STD-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
-; SI-STD-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
@@ -1070,21 +1070,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-STD-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-STD-NEXT: s_cbranch_vccnz .LBB12_2
; SI-STD-NEXT: ; %bb.1: ; %normal
-; SI-STD-NEXT: v_mul_f32_e32 v4, v6, v1
-; SI-STD-NEXT: v_fma_f32 v4, v2, v3, v4
-; SI-STD-NEXT: v_sub_f32_e32 v4, v4, v5
+; SI-STD-NEXT: v_mul_f32_e32 v2, v6, v1
+; SI-STD-NEXT: v_fma_f32 v2, v3, v4, v2
+; SI-STD-NEXT: v_sub_f32_e32 v2, v2, v5
; SI-STD-NEXT: s_cbranch_execz .LBB12_3
; SI-STD-NEXT: s_branch .LBB12_4
; SI-STD-NEXT: .LBB12_2:
-; SI-STD-NEXT: ; implicit-def: $vgpr4
+; SI-STD-NEXT: ; implicit-def: $vgpr2
; SI-STD-NEXT: .LBB12_3: ; %aggressive
-; SI-STD-NEXT: v_mad_f32 v4, v6, v1, -v5
-; SI-STD-NEXT: v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT: v_mad_f32 v2, v6, v1, -v5
+; SI-STD-NEXT: v_mac_f32_e32 v2, v3, v4
; SI-STD-NEXT: .LBB12_4: ; %exit
; SI-STD-NEXT: s_mov_b32 s3, 0xf000
; SI-STD-NEXT: s_mov_b32 s2, 0
; SI-STD-NEXT: v_mov_b32_e32 v1, 0
-; SI-STD-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-STD-NEXT: s_endpgm
;
; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_0_f32:
@@ -1099,7 +1099,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
@@ -1112,21 +1112,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_vccnz .LBB12_2
; SI-DENORM-FASTFMAF-NEXT: ; %bb.1: ; %normal
-; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v4, v6, v1
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v4, v2, v3, v4
-; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v4, v4, v5
+; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v3, v6, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v3, v2, v4, v3
+; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v3, v3, v5
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_execz .LBB12_3
; SI-DENORM-FASTFMAF-NEXT: s_branch .LBB12_4
; SI-DENORM-FASTFMAF-NEXT: .LBB12_2:
-; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr4
+; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr3
; SI-DENORM-FASTFMAF-NEXT: .LBB12_3: ; %aggressive
; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v1, v6, v1, -v5
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v4, v2, v3, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v3, v2, v4, v1
; SI-DENORM-FASTFMAF-NEXT: .LBB12_4: ; %exit
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s3, 0xf000
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s2, 0
; SI-DENORM-FASTFMAF-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-DENORM-FASTFMAF-NEXT: s_endpgm
;
; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_0_f32:
@@ -1139,11 +1139,11 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt lgkmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-DENORM-SLOWFMAF-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
@@ -1153,21 +1153,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
; SI-DENORM-SLOWFMAF-NEXT: s_cselect_b64 s[2:3], -1, 0
; SI-DENORM-SLOWFMAF-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v1, v5, v1
-; SI-DENORM-SLOWFMAF-NEXT: v_fma_f32 v1, v3, v4, v1
+; SI-DENORM-SLOWFMAF-NEXT: v_fma_f32 v1, v2, v4, v1
; SI-DENORM-SLOWFMAF-NEXT: s_cbranch_vccnz .LBB12_2
; SI-DENORM-SLOWFMAF-NEXT: ; %bb.1: ; %normal
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v1, v3
; SI-DENORM-SLOWFMAF-NEXT: s_cbranch_execz .LBB12_3
; SI-DENORM-SLOWFMAF-NEXT: s_branch .LBB12_4
; SI-DENORM-SLOWFMAF-NEXT: .LBB12_2:
-; SI-DENORM-SLOWFMAF-NEXT: ; implicit-def: $vgpr3
+; SI-DENORM-SLOWFMAF-NEXT: ; implicit-def: $vgpr2
; SI-DENORM-SLOWFMAF-NEXT: .LBB12_3: ; %aggressive
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v1, v3
; SI-DENORM-SLOWFMAF-NEXT: .LBB12_4: ; %exit
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b32 s3, 0xf000
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b32 s2, 0
; SI-DENORM-SLOWFMAF-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMAF-NEXT: s_endpgm
%tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
%gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
@@ -1267,7 +1267,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-STD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-STD-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
-; SI-STD-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-STD-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
@@ -1280,21 +1280,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-STD-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-STD-NEXT: s_cbranch_vccnz .LBB14_2
; SI-STD-NEXT: ; %bb.1: ; %normal
-; SI-STD-NEXT: v_mul_f32_e32 v4, v6, v1
-; SI-STD-NEXT: v_mac_f32_e32 v4, v2, v3
-; SI-STD-NEXT: v_sub_f32_e32 v4, v4, v5
+; SI-STD-NEXT: v_mul_f32_e32 v3, v6, v1
+; SI-STD-NEXT: v_mac_f32_e32 v3, v2, v4
+; SI-STD-NEXT: v_sub_f32_e32 v3, v3, v5
; SI-STD-NEXT: s_cbranch_execz .LBB14_3
; SI-STD-NEXT: s_branch .LBB14_4
; SI-STD-NEXT: .LBB14_2:
-; SI-STD-NEXT: ; implicit-def: $vgpr4
+; SI-STD-NEXT: ; implicit-def: $vgpr3
; SI-STD-NEXT: .LBB14_3: ; %aggressive
-; SI-STD-NEXT: v_mad_f32 v4, v6, v1, -v5
-; SI-STD-NEXT: v_mac_f32_e32 v4, v2, v3
+; SI-STD-NEXT: v_mad_f32 v3, v6, v1, -v5
+; SI-STD-NEXT: v_mac_f32_e32 v3, v2, v4
; SI-STD-NEXT: .LBB14_4: ; %exit
; SI-STD-NEXT: s_mov_b32 s3, 0xf000
; SI-STD-NEXT: s_mov_b32 s2, 0
; SI-STD-NEXT: v_mov_b32_e32 v1, 0
-; SI-STD-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-STD-NEXT: s_endpgm
;
; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_2_f32:
@@ -1309,7 +1309,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
@@ -1322,21 +1322,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_vccnz .LBB14_2
; SI-DENORM-FASTFMAF-NEXT: ; %bb.1: ; %normal
-; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v4, v6, v1
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v4, v2, v3, v4
-; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v4, v4, v5
+; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v3, v6, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v3, v2, v4, v3
+; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v3, v3, v5
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_execz .LBB14_3
; SI-DENORM-FASTFMAF-NEXT: s_branch .LBB14_4
; SI-DENORM-FASTFMAF-NEXT: .LBB14_2:
-; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr4
+; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr3
; SI-DENORM-FASTFMAF-NEXT: .LBB14_3: ; %aggressive
; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v1, v6, v1, -v5
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v4, v2, v3, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v3, v2, v4, v1
; SI-DENORM-FASTFMAF-NEXT: .LBB14_4: ; %exit
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s3, 0xf000
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s2, 0
; SI-DENORM-FASTFMAF-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-DENORM-FASTFMAF-NEXT: s_endpgm
;
; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_2_f32:
@@ -1349,11 +1349,11 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt lgkmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-DENORM-SLOWFMAF-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-SLOWFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-DENORM-SLOWFMAF-NEXT: s_waitcnt vmcnt(0)
@@ -1362,23 +1362,23 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
; SI-DENORM-SLOWFMAF-NEXT: s_bitcmp1_b32 s8, 0
; SI-DENORM-SLOWFMAF-NEXT: s_cselect_b64 s[2:3], -1, 0
; SI-DENORM-SLOWFMAF-NEXT: s_and_b64 vcc, exec, s[2:3]
-; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v3, v3, v4
+; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v2, v2, v4
; SI-DENORM-SLOWFMAF-NEXT: v_mul_f32_e32 v1, v5, v1
-; SI-DENORM-SLOWFMAF-NEXT: v_add_f32_e32 v1, v3, v1
+; SI-DENORM-SLOWFMAF-NEXT: v_add_f32_e32 v1, v2, v1
; SI-DENORM-SLOWFMAF-NEXT: s_cbranch_vccnz .LBB14_2
; SI-DENORM-SLOWFMAF-NEXT: ; %bb.1: ; %normal
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v1, v3
; SI-DENORM-SLOWFMAF-NEXT: s_cbranch_execz .LBB14_3
; SI-DENORM-SLOWFMAF-NEXT: s_branch .LBB14_4
; SI-DENORM-SLOWFMAF-NEXT: .LBB14_2:
-; SI-DENORM-SLOWFMAF-NEXT: ; implicit-def: $vgpr3
+; SI-DENORM-SLOWFMAF-NEXT: ; implicit-def: $vgpr2
; SI-DENORM-SLOWFMAF-NEXT: .LBB14_3: ; %aggressive
-; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v3, v1, v2
+; SI-DENORM-SLOWFMAF-NEXT: v_sub_f32_e32 v2, v1, v3
; SI-DENORM-SLOWFMAF-NEXT: .LBB14_4: ; %exit
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b32 s3, 0xf000
; SI-DENORM-SLOWFMAF-NEXT: s_mov_b32 s2, 0
; SI-DENORM-SLOWFMAF-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-SLOWFMAF-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-SLOWFMAF-NEXT: s_endpgm
%tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
%gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
@@ -1431,7 +1431,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
-; SI-STD-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-STD-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
; SI-STD-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-STD-NEXT: s_waitcnt vmcnt(0)
@@ -1442,21 +1442,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
; SI-STD-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-STD-NEXT: s_cbranch_vccnz .LBB15_2
; SI-STD-NEXT: ; %bb.1: ; %normal
-; SI-STD-NEXT: v_mul_f32_e32 v5, v6, v1
-; SI-STD-NEXT: v_mac_f32_e32 v5, v3, v4
-; SI-STD-NEXT: v_sub_f32_e32 v5, v2, v5
+; SI-STD-NEXT: v_mul_f32_e32 v4, v6, v1
+; SI-STD-NEXT: v_mac_f32_e32 v4, v3, v5
+; SI-STD-NEXT: v_sub_f32_e32 v4, v2, v4
; SI-STD-NEXT: s_cbranch_execz .LBB15_3
; SI-STD-NEXT: s_branch .LBB15_4
; SI-STD-NEXT: .LBB15_2:
-; SI-STD-NEXT: ; implicit-def: $vgpr5
+; SI-STD-NEXT: ; implicit-def: $vgpr4
; SI-STD-NEXT: .LBB15_3: ; %aggressive
; SI-STD-NEXT: v_mad_f32 v1, -v6, v1, v2
-; SI-STD-NEXT: v_mad_f32 v5, -v3, v4, v1
+; SI-STD-NEXT: v_mad_f32 v4, -v3, v5, v1
; SI-STD-NEXT: .LBB15_4: ; %exit
; SI-STD-NEXT: s_mov_b32 s3, 0xf000
; SI-STD-NEXT: s_mov_b32 s2, 0
; SI-STD-NEXT: v_mov_b32_e32 v1, 0
-; SI-STD-NEXT: buffer_store_dword v5, v[0:1], s[0:3], 0 addr64
+; SI-STD-NEXT: buffer_store_dword v4, v[0:1], s[0:3], 0 addr64
; SI-STD-NEXT: s_endpgm
;
; SI-DENORM-FASTFMAF-LABEL: aggressive_combine_to_mad_fsub_3_f32:
@@ -1469,11 +1469,11 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt lgkmcnt(0)
; SI-DENORM-FASTFMAF-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-DENORM-FASTFMAF-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 glc
+; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:4 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
-; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v5, v[0:1], s[4:7], 0 addr64 offset:8 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
; SI-DENORM-FASTFMAF-NEXT: buffer_load_dword v6, v[0:1], s[4:7], 0 addr64 offset:12 glc
; SI-DENORM-FASTFMAF-NEXT: s_waitcnt vmcnt(0)
@@ -1484,21 +1484,21 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
; SI-DENORM-FASTFMAF-NEXT: s_and_b64 vcc, exec, s[2:3]
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_vccnz .LBB15_2
; SI-DENORM-FASTFMAF-NEXT: ; %bb.1: ; %normal
-; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v5, v6, v1
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v5, v3, v4, v5
-; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v5, v2, v5
+; SI-DENORM-FASTFMAF-NEXT: v_mul_f32_e32 v2, v6, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v2, v4, v5, v2
+; SI-DENORM-FASTFMAF-NEXT: v_sub_f32_e32 v2, v3, v2
; SI-DENORM-FASTFMAF-NEXT: s_cbranch_execz .LBB15_3
; SI-DENORM-FASTFMAF-NEXT: s_branch .LBB15_4
; SI-DENORM-FASTFMAF-NEXT: .LBB15_2:
-; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr5
+; SI-DENORM-FASTFMAF-NEXT: ; implicit-def: $vgpr2
; SI-DENORM-FASTFMAF-NEXT: .LBB15_3: ; %aggressive
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v1, -v6, v1, v2
-; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v5, -v3, v4, v1
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v1, -v6, v1, v3
+; SI-DENORM-FASTFMAF-NEXT: v_fma_f32 v2, -v4, v5, v1
; SI-DENORM-FASTFMAF-NEXT: .LBB15_4: ; %exit
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s3, 0xf000
; SI-DENORM-FASTFMAF-NEXT: s_mov_b32 s2, 0
; SI-DENORM-FASTFMAF-NEXT: v_mov_b32_e32 v1, 0
-; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v5, v[0:1], s[0:3], 0 addr64
+; SI-DENORM-FASTFMAF-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-DENORM-FASTFMAF-NEXT: s_endpgm
;
; SI-DENORM-SLOWFMAF-LABEL: aggressive_combine_to_mad_fsub_3_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
index d95965caa81ab..c353b6962fb8f 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-var-size.ll
@@ -1043,55 +1043,55 @@ define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p1_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB7_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB7_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[10:13], v9
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: ds_read_b128 v[9:12], v8
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_2
; CHECK-NEXT: .LBB7_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB7_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; CHECK-NEXT: .LBB7_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_u8 v7, v2
+; CHECK-NEXT: ds_read_u8 v5, v2
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB7_5
; CHECK-NEXT: .LBB7_6: ; %Flow7
@@ -1252,60 +1252,60 @@ define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p1_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB9_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v2
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB9_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s4
+; CHECK-NEXT: buffer_load_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: v_add_co_u32 v13, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo
+; CHECK-NEXT: v_add_co_ci_u32_e64 v14, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off
+; CHECK-NEXT: global_store_dwordx4 v[13:14], v[9:12], off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_2
; CHECK-NEXT: .LBB9_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB9_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; CHECK-NEXT: .LBB9_5: ; %dynamic-memcpy-expansion-residual-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_byte v[3:4], v7, off
+; CHECK-NEXT: global_store_byte v[3:4], v5, off
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB9_5
; CHECK-NEXT: .LBB9_6: ; %Flow7
@@ -1465,39 +1465,39 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p3_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB11_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB11_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB11_2
; CHECK-NEXT: .LBB11_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB11_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1508,7 +1508,7 @@ define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1529,13 +1529,13 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-LABEL: memmove_p3_p3:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v2
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
; CHECK-NEXT: v_cmpx_ge_u32_e64 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_3
@@ -1550,16 +1550,16 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB12_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .LBB12_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[9:12], v3
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: ds_read_b128 v[9:12], v7
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b128 v8, v[9:12]
@@ -1578,10 +1578,10 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: .LBB12_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v2, v1
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: ds_write_b8 v0, v2
@@ -1593,7 +1593,8 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB12_2
; CHECK-NEXT: .LBB12_10: ; %memmove_copy_backwards
@@ -1602,19 +1603,19 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB12_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ds_read_u8 v8, v7
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b8 v6, v8
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: ds_write_b8 v3, v8
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -1, v3
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB12_12
; CHECK-NEXT: .LBB12_13: ; %Flow45
@@ -1624,21 +1625,21 @@ define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v5
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v3
; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
-; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v1, v3
+; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v4, vcc_lo
; CHECK-NEXT: .LBB12_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ds_read_b128 v[5:8], v4
+; CHECK-NEXT: ds_read_b128 v[4:7], v3
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v3
; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; CHECK-NEXT: s_or_b32 s6, vcc_lo, s6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: ds_write_b128 v2, v[5:8]
+; CHECK-NEXT: ds_write_b128 v2, v[4:7]
; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB12_15
@@ -1656,39 +1657,39 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p3_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB13_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .LBB13_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: ds_write_b128 v9, v[10:13]
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: ds_write_b128 v8, v[9:12]
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB13_2
; CHECK-NEXT: .LBB13_3: ; %Flow9
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB13_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1699,7 +1700,7 @@ define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -1936,32 +1937,32 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-LABEL: memmove_p5_p1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB16_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB16_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB16_2
@@ -1969,10 +1970,10 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB16_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -1983,7 +1984,7 @@ define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2068,32 +2069,32 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-LABEL: memmove_p5_p4:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v7, -16, v3
-; CHECK-NEXT: v_mov_b32_e32 v8, v4
-; CHECK-NEXT: v_and_b32_e32 v5, 15, v3
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, v3
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
+; CHECK-NEXT: v_and_b32_e32 v6, 15, v5
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[3:4]
; CHECK-NEXT: s_cbranch_execz .LBB18_3
; CHECK-NEXT: ; %bb.1: ; %dynamic-memcpy-expansion-main-body.preheader
-; CHECK-NEXT: v_mov_b32_e32 v9, v0
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB18_2: ; %dynamic-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v9, vcc_lo, v1, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 16
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]
-; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[3:4]
+; CHECK-NEXT: global_load_dwordx4 v[9:12], v[9:10], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v9
+; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB18_2
@@ -2101,10 +2102,10 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]
+; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB18_6
; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body.preheader
-; CHECK-NEXT: v_and_b32_e32 v3, -16, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v5
; CHECK-NEXT: s_mov_b32 s7, 0
; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v3
@@ -2115,7 +2116,7 @@ define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align
; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]
+; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[6:7]
; CHECK-NEXT: global_load_ubyte v3, v[3:4], off
; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -2135,13 +2136,13 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-LABEL: memmove_p5_p5:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_and_b32_e32 v4, 15, v2
-; CHECK-NEXT: v_and_b32_e32 v6, -16, v2
-; CHECK-NEXT: v_mov_b32_e32 v7, v3
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_and_b32_e32 v5, 15, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_and_b32_e32 v3, -16, v2
; CHECK-NEXT: s_mov_b32 s6, exec_lo
-; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]
-; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]
+; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[3:4]
; CHECK-NEXT: v_cmpx_ge_u32_e64 v1, v0
; CHECK-NEXT: s_xor_b32 s6, exec_lo, s6
; CHECK-NEXT: s_cbranch_execnz .LBB19_3
@@ -2155,20 +2156,20 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB19_6
; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
+; CHECK-NEXT: v_mov_b32_e32 v7, v1
; CHECK-NEXT: v_mov_b32_e32 v8, v0
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_5: ; %memmove_fwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v9, v3, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v10, v3, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v11, v3, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v12, v3, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -16
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v3
+; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v12, v7, s[0:3], 0 offen
+; CHECK-NEXT: v_add_co_u32 v3, s5, v3, -16
+; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v4, s5
+; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v7
; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
@@ -2177,7 +2178,7 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[3:4]
; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v8
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
@@ -2194,10 +2195,10 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: .LBB19_8: ; %memmove_fwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5
+; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s5
; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen
@@ -2209,7 +2210,8 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
; CHECK-NEXT: ; implicit-def: $vgpr0
; CHECK-NEXT: ; implicit-def: $vgpr1
-; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6
; CHECK-NEXT: s_andn2_saveexec_b32 s5, s6
; CHECK-NEXT: s_cbranch_execz .LBB19_2
; CHECK-NEXT: .LBB19_10: ; %memmove_copy_backwards
@@ -2218,19 +2220,19 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v2
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v7
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v7
; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v7
; CHECK-NEXT: .LBB19_12: ; %memmove_bwd_residual_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: buffer_load_ubyte v8, v7, s[0:3], 0 offen
-; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1
-; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4
+; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -1
+; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s4
; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v7
-; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]
+; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[5:6]
; CHECK-NEXT: s_or_b32 s7, s4, s7
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_byte v8, v6, s[0:3], 0 offen
-; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; CHECK-NEXT: buffer_store_byte v8, v3, s[0:3], 0 offen
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -1, v3
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7
; CHECK-NEXT: s_cbranch_execnz .LBB19_12
; CHECK-NEXT: .LBB19_13: ; %Flow45
@@ -2240,24 +2242,24 @@ define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align
; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader
; CHECK-NEXT: v_and_b32_e32 v5, -16, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v5
+; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v3
; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v4
-; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; CHECK-NEXT: v_add_nc_u32_e32 v3, v1, v3
+; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v4, vcc_lo
; CHECK-NEXT: .p2align 6
; CHECK-NEXT: .LBB19_15: ; %memmove_bwd_main_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v3, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v5, v3, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v6, v3, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v7, v3, s[0:3], 0 offen
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 16
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v4
+; CHECK-NEXT: v_add_nc_u32_e32 v3, -16, v3
; CHECK-NEXT: s_waitcnt vmcnt(3)
-; CHECK-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:12
+; CHECK-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:12
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 990a986ffab75..d12ce0e1f38f0 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -7,14 +7,13 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p0_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v12
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v12
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -29,7 +28,7 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
@@ -41,7 +40,7 @@ define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v12
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
@@ -132,14 +131,13 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-LABEL: memset_p1_varsize_align_4_varsetval:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v3
; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v12
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v12
; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
@@ -154,7 +152,7 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
@@ -166,7 +164,7 @@ define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst,
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v12
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
@@ -259,35 +257,32 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v0
+; GFX942-SDAG-NEXT: v_perm_b32 v3, v1, v1, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v8, v3, v3 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v8, v3, v3 offset1:1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: v_add_u32_e32 v9, 16, v9
+; GFX942-SDAG-NEXT: v_add_u32_e32 v8, 16, v8
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB2_2
; GFX942-SDAG-NEXT: .LBB2_3: ; %Flow7
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
@@ -298,7 +293,7 @@ define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst,
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT: ds_write_b8 v0, v1
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
@@ -500,52 +495,10 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
@@ -556,41 +509,41 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-SDAG-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[38:39], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[10:11], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[20:23] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, 48
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[12:15] offset:32
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[8:11] offset:16
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[60:61], v[38:39], 0, s[20:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7] offset:128
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[32:35]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[28:31]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[32:35]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[28:31]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[24:27]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[20:23]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[16:19]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[12:15]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[60:61], v[8:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[10:11], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[10:11], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[18:19], v[10:11], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[20:21], v[10:11], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[22:23], v[10:11], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[24:25], v[10:11], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[26:27], v[10:11], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[28:29], v[10:11], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[30:31], v[10:11], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[32:33], v[10:11], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[18:19], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[20:21], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[22:23], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[24:25], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[26:27], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[28:29], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[30:31], v[4:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[32:33], v[4:7]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB4_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -604,20 +557,6 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
; GFX942-SDAG-NEXT: flat_store_short v[0:1], v3 offset:1052
; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 offset:1054
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -692,50 +631,10 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
@@ -746,55 +645,43 @@ define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[36:37], 0, s[2:3]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[8:9], 0, s[2:3]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[18:21] offset:64
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, 48
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[10:13] offset:32
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[6:9] offset:16
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[8:9]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[10:11]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[12:13]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[14:15]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[16:17]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[18:19]
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[36:37], 0, s[20:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5] offset:128
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[30:33]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[26:29]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[22:25]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[18:21]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[14:17]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[10:13]
-; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[6:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[8:9], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[8:9], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[16:17], v[8:9], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[18:19], v[8:9], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[20:21], v[8:9], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[22:23], v[8:9], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[24:25], v[8:9], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[26:27], v[8:9], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[28:29], v[8:9], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[30:31], v[8:9], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[10:11], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[12:13], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[16:17], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[18:19], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[20:21], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[22:23], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[24:25], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[26:27], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[28:29], v[2:5]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[30:31], v[2:5]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB5_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -851,62 +738,34 @@ define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], 0x400
; GFX942-SDAG-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:112
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:96
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:80
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:64
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:240
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:224
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:208
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:192
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:176
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:160
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:144
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off offset:128
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:128
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB6_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -994,62 +853,34 @@ define void @memset_p1_sz2048_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[8:9], v[0:1], 0, s[0:1]
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:112
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:96
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:80
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:64
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:240
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:224
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:208
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:192
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:176
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:160
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:144
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off offset:128
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:128
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB7_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -1107,79 +938,48 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v0
; GFX942-SDAG-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset0:32 offset1:33
-; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v5, v4, v4 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_add_u32_e32 v5, 0x100, v5
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB8_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -1268,77 +1068,46 @@ define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_perm_b32 v1, v1, v1, s0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x800
; GFX942-SDAG-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:30 offset1:31
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:28 offset1:29
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:26 offset1:27
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:24 offset1:25
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:22 offset1:23
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:20 offset1:21
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:18 offset1:19
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:16 offset1:17
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:14 offset1:15
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:12 offset1:13
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:10 offset1:11
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:8 offset1:9
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:6 offset1:7
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:4 offset1:5
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:2 offset1:3
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:62 offset1:63
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:60 offset1:61
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:58 offset1:59
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:56 offset1:57
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:54 offset1:55
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:52 offset1:53
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:50 offset1:51
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:48 offset1:49
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:46 offset1:47
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:44 offset1:45
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:42 offset1:43
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:40 offset1:41
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:38 offset1:39
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:36 offset1:37
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:34 offset1:35
-; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v1, v1 offset0:32 offset1:33
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB9_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
@@ -1396,62 +1165,34 @@ define void @memset_p5_sz1055_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v0
; GFX942-SDAG-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:112
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:96
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:80
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:64
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:48
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:32
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:16
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:240
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:224
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:208
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:192
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:176
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:160
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:144
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off offset:128
-; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:16
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:128
+; GFX942-SDAG-NEXT: v_add_u32_e32 v8, 0x100, v8
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB10_1
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
@@ -1541,59 +1282,31 @@ define void @memset_p5_sz2048_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], 0x800
; GFX942-SDAG-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:112
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:96
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:80
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:64
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:16
; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:240
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:224
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:208
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:192
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:176
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:160
-; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:144
; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:128
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB11_1
@@ -1650,38 +1363,38 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0x28282828
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, 0x28282828
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v6
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[6:9], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v4
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
@@ -1690,7 +1403,7 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
@@ -1770,26 +1483,26 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
; GFX942-SDAG: ; %bb.0: ; %entry
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, 15, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v7
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v7
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v7
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v7
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
@@ -1797,22 +1510,22 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v4
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-SDAG-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow2
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index a881b83986f8d..f1332863dabe4 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -994,37 +994,9 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: v_add_u32_e32 v0, s16, v0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v1
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v3
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v1
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v3
-; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v0
; GFX942-SDAG-NEXT: .LBB14_1: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1032,21 +1004,21 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
; GFX942-SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:16
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:32
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:48
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:64
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:80
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:96
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:112
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:16
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:32
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:48
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:64
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:80
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:96
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:112
; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:128
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[6:9], v1, s[0:3], 0 offen offset:144
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[10:13], v1, s[0:3], 0 offen offset:160
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[14:17], v1, s[0:3], 0 offen offset:176
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[18:21], v1, s[0:3], 0 offen offset:192
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[22:25], v1, s[0:3], 0 offen offset:208
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[26:29], v1, s[0:3], 0 offen offset:224
-; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[30:33], v1, s[0:3], 0 offen offset:240
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:144
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:160
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:176
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:192
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:208
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:224
+; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen offset:240
; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 0x2000, v1
; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB14_1
diff --git a/llvm/test/CodeGen/AMDGPU/merge-stores.ll b/llvm/test/CodeGen/AMDGPU/merge-stores.ll
index 7457509ffe193..bf06b26874fe9 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/merge-stores.ll
@@ -1226,25 +1226,25 @@ define amdgpu_kernel void @copy_v3f32_align4(ptr addrspace(1) noalias %out, ptr
define amdgpu_kernel void @copy_v3f64_align4(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #0 {
; GCN-LABEL: copy_v3f64_align4:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s10, s6
-; GCN-NEXT: s_mov_b32 s11, s7
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s10, s2
+; GCN-NEXT: s_mov_b32 s11, s3
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s8, s2
-; GCN-NEXT: s_mov_b32 s9, s3
+; GCN-NEXT: s_mov_b32 s8, s6
+; GCN-NEXT: s_mov_b32 s9, s7
; GCN-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; GCN-NEXT: buffer_load_dwordx2 v[4:5], off, s[8:11], 0 offset:16
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_mov_b32 s0, s4
+; GCN-NEXT: s_mov_b32 s1, s5
; GCN-NEXT: s_waitcnt vmcnt(1)
; GCN-NEXT: v_add_f64 v[2:3], v[2:3], 2.0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_add_f64 v[4:5], v[4:5], 4.0
; GCN-NEXT: v_add_f64 v[0:1], v[0:1], 1.0
-; GCN-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 0 offset:16
-; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT: buffer_store_dwordx2 v[4:5], off, s[0:3], 0 offset:16
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GCN-NEXT: s_endpgm
%vec = load <3 x double>, ptr addrspace(1) %in, align 4
%fadd = fadd <3 x double> %vec, <double 1.0, double 2.0, double 4.0>
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index 263298d1cd1c8..09987c1065a53 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -295,99 +295,105 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr
; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def a0
; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_mov_b32_e32 v26, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v25, 2.0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX908-NEXT: global_load_dwordx4 v[1:4], v0, s[0:1]
+; GFX908-NEXT: global_load_dwordx4 v[5:8], v0, s[0:1] offset:16
+; GFX908-NEXT: global_load_dwordx4 v[9:12], v0, s[0:1] offset:32
+; GFX908-NEXT: global_load_dwordx4 v[13:16], v0, s[0:1] offset:48
+; GFX908-NEXT: global_load_dwordx4 v[17:20], v0, s[0:1] offset:64
+; GFX908-NEXT: global_load_dwordx4 v[21:24], v0, s[0:1] offset:80
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v4
+; GFX908-NEXT: global_load_dwordx4 v[1:4], v0, s[0:1] offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v8
+; GFX908-NEXT: global_load_dwordx4 v[5:8], v0, s[0:1] offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v9
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v10
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v11
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v12
+; GFX908-NEXT: s_waitcnt vmcnt(4)
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v13
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v14
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v15
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v16
+; GFX908-NEXT: s_waitcnt vmcnt(3)
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v17
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v18
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v19
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v20
+; GFX908-NEXT: s_waitcnt vmcnt(2)
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v21
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v22
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v23
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v24
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v6
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v7
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v8
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v9
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v10
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v11
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v12
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v13
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v14
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v15
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v16
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v17
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v18
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v19
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v20
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v21
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v22
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v23
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v24
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v31
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v8
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v26, v25, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a12
+; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[0:1] offset:96
+; GFX908-NEXT: global_store_dwordx4 v0, v[5:8], s[0:1] offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a4
+; GFX908-NEXT: global_store_dwordx4 v0, v[9:12], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v0, v[13:16], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v0, v[17:20], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v0, v[21:24], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v0, v[5:8], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
%acc = call i32 asm sideeffect "; def $0", "={a0}"()
@@ -401,100 +407,106 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add
; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_phys_agpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: ; implicit-def: $agpr100_agpr101_agpr102_agpr103_agpr104_agpr105_agpr106_agpr107_agpr108_agpr109_agpr110_agpr111_agpr112_agpr113_agpr114_agpr115_agpr116_agpr117_agpr118_agpr119_agpr120_agpr121_agpr122_agpr123_agpr124_agpr125_agpr126_agpr127_agpr128_agpr129_agpr130_agpr131
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; use a[100:131]
; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_mov_b32_e32 v26, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v25, 2.0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GFX908-NEXT: global_load_dwordx4 v[1:4], v0, s[0:1]
+; GFX908-NEXT: global_load_dwordx4 v[5:8], v0, s[0:1] offset:16
+; GFX908-NEXT: global_load_dwordx4 v[9:12], v0, s[0:1] offset:32
+; GFX908-NEXT: global_load_dwordx4 v[13:16], v0, s[0:1] offset:48
+; GFX908-NEXT: global_load_dwordx4 v[17:20], v0, s[0:1] offset:64
+; GFX908-NEXT: global_load_dwordx4 v[21:24], v0, s[0:1] offset:80
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v4
+; GFX908-NEXT: global_load_dwordx4 v[1:4], v0, s[0:1] offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v8
+; GFX908-NEXT: global_load_dwordx4 v[5:8], v0, s[0:1] offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v9
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v10
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v11
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v12
+; GFX908-NEXT: s_waitcnt vmcnt(4)
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v13
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v14
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v15
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v16
+; GFX908-NEXT: s_waitcnt vmcnt(3)
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v17
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v18
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v19
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v20
+; GFX908-NEXT: s_waitcnt vmcnt(2)
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v21
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v22
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v23
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v24
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v6
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v7
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v8
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v9
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v10
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v11
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v12
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v13
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v14
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v15
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v16
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v17
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v18
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v19
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v20
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v21
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v22
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v23
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v24
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v31
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v8
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v26, v25, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a26
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a25
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a24
-; GFX908-NEXT: v_accvgpr_read_b32 v7, a31
-; GFX908-NEXT: v_accvgpr_read_b32 v6, a30
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a29
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a28
-; GFX908-NEXT: v_accvgpr_read_b32 v11, a19
-; GFX908-NEXT: v_accvgpr_read_b32 v10, a18
-; GFX908-NEXT: v_accvgpr_read_b32 v9, a17
-; GFX908-NEXT: v_accvgpr_read_b32 v8, a16
-; GFX908-NEXT: v_accvgpr_read_b32 v15, a23
-; GFX908-NEXT: v_accvgpr_read_b32 v14, a22
-; GFX908-NEXT: v_accvgpr_read_b32 v13, a21
-; GFX908-NEXT: v_accvgpr_read_b32 v12, a20
-; GFX908-NEXT: v_accvgpr_read_b32 v19, a11
-; GFX908-NEXT: v_accvgpr_read_b32 v18, a10
-; GFX908-NEXT: v_accvgpr_read_b32 v17, a9
-; GFX908-NEXT: v_accvgpr_read_b32 v16, a8
-; GFX908-NEXT: v_accvgpr_read_b32 v23, a15
-; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
-; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
-; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a27
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a26
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a25
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a24
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a31
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a30
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a29
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a28
+; GFX908-NEXT: v_accvgpr_read_b32 v12, a19
+; GFX908-NEXT: v_accvgpr_read_b32 v11, a18
+; GFX908-NEXT: v_accvgpr_read_b32 v10, a17
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a16
+; GFX908-NEXT: v_accvgpr_read_b32 v16, a23
+; GFX908-NEXT: v_accvgpr_read_b32 v15, a22
+; GFX908-NEXT: v_accvgpr_read_b32 v14, a21
+; GFX908-NEXT: v_accvgpr_read_b32 v13, a20
+; GFX908-NEXT: v_accvgpr_read_b32 v20, a11
+; GFX908-NEXT: v_accvgpr_read_b32 v19, a10
+; GFX908-NEXT: v_accvgpr_read_b32 v18, a9
+; GFX908-NEXT: v_accvgpr_read_b32 v17, a8
+; GFX908-NEXT: v_accvgpr_read_b32 v24, a15
+; GFX908-NEXT: v_accvgpr_read_b32 v23, a14
+; GFX908-NEXT: v_accvgpr_read_b32 v22, a13
+; GFX908-NEXT: v_accvgpr_read_b32 v21, a12
+; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[0:1] offset:96
+; GFX908-NEXT: global_store_dwordx4 v0, v[5:8], s[0:1] offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a4
+; GFX908-NEXT: global_store_dwordx4 v0, v[9:12], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v0, v[13:16], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v0, v[17:20], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v0, v[21:24], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v0, v[1:4], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v0, v[5:8], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
call void asm sideeffect "; use $0", "{a[100:131]}"(<32 x float> poison)
@@ -508,56 +520,63 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr
; GFX908-LABEL: test_mfma_f32_32x32x1f32_inline_asm_no_agprs:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX908-NEXT: v_mov_b32_e32 v32, 0
+; GFX908-NEXT: v_mov_b32_e32 v24, 0
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ; def v0
; GFX908-NEXT: ;;#ASMEND
+; GFX908-NEXT: v_mov_b32_e32 v26, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v25, 2.0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
-; GFX908-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GFX908-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GFX908-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GFX908-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GFX908-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GFX908-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GFX908-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1]
+; GFX908-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:16
+; GFX908-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:32
+; GFX908-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:48
+; GFX908-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:64
+; GFX908-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:80
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX908-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
; GFX908-NEXT: v_accvgpr_write_b32 a6, v6
; GFX908-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX908-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a8, v8
; GFX908-NEXT: v_accvgpr_write_b32 a9, v9
; GFX908-NEXT: v_accvgpr_write_b32 a10, v10
; GFX908-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX908-NEXT: s_waitcnt vmcnt(4)
; GFX908-NEXT: v_accvgpr_write_b32 a12, v12
; GFX908-NEXT: v_accvgpr_write_b32 a13, v13
; GFX908-NEXT: v_accvgpr_write_b32 a14, v14
; GFX908-NEXT: v_accvgpr_write_b32 a15, v15
+; GFX908-NEXT: s_waitcnt vmcnt(3)
; GFX908-NEXT: v_accvgpr_write_b32 a16, v16
; GFX908-NEXT: v_accvgpr_write_b32 a17, v17
; GFX908-NEXT: v_accvgpr_write_b32 a18, v18
; GFX908-NEXT: v_accvgpr_write_b32 a19, v19
+; GFX908-NEXT: s_waitcnt vmcnt(2)
; GFX908-NEXT: v_accvgpr_write_b32 a20, v20
; GFX908-NEXT: v_accvgpr_write_b32 a21, v21
; GFX908-NEXT: v_accvgpr_write_b32 a22, v22
; GFX908-NEXT: v_accvgpr_write_b32 a23, v23
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v24
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v31
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v7
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v26, v25, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
@@ -584,23 +603,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr
; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
-; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112
-; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64
-; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80
-; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
-; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48
-; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1]
-; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
+; GFX908-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1] offset:96
+; GFX908-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
+; GFX908-NEXT: global_store_dwordx4 v24, v[8:11], s[0:1] offset:64
+; GFX908-NEXT: global_store_dwordx4 v24, v[12:15], s[0:1] offset:80
+; GFX908-NEXT: global_store_dwordx4 v24, v[16:19], s[0:1] offset:32
+; GFX908-NEXT: global_store_dwordx4 v24, v[20:23], s[0:1] offset:48
+; GFX908-NEXT: global_store_dwordx4 v24, v[0:3], s[0:1]
+; GFX908-NEXT: global_store_dwordx4 v24, v[4:7], s[0:1] offset:16
; GFX908-NEXT: s_endpgm
bb:
%acc = call i32 asm sideeffect "; def $0", "={v0}"()
@@ -642,51 +660,58 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: ; implicit-def: $sgpr15
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX908-NEXT: global_load_dwordx4 v[28:31], v40, s[34:35] offset:112
-; GFX908-NEXT: global_load_dwordx4 v[24:27], v40, s[34:35] offset:96
-; GFX908-NEXT: global_load_dwordx4 v[20:23], v40, s[34:35] offset:80
-; GFX908-NEXT: global_load_dwordx4 v[16:19], v40, s[34:35] offset:64
-; GFX908-NEXT: global_load_dwordx4 v[12:15], v40, s[34:35] offset:48
-; GFX908-NEXT: global_load_dwordx4 v[8:11], v40, s[34:35] offset:32
-; GFX908-NEXT: global_load_dwordx4 v[4:7], v40, s[34:35] offset:16
; GFX908-NEXT: global_load_dwordx4 v[0:3], v40, s[34:35]
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: global_load_dwordx4 v[4:7], v40, s[34:35] offset:16
+; GFX908-NEXT: global_load_dwordx4 v[8:11], v40, s[34:35] offset:32
+; GFX908-NEXT: global_load_dwordx4 v[12:15], v40, s[34:35] offset:48
+; GFX908-NEXT: global_load_dwordx4 v[16:19], v40, s[34:35] offset:64
+; GFX908-NEXT: global_load_dwordx4 v[20:23], v40, s[34:35] offset:80
+; GFX908-NEXT: v_mov_b32_e32 v25, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v24, 2.0
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX908-NEXT: global_load_dwordx4 v[0:3], v40, s[34:35] offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
; GFX908-NEXT: v_accvgpr_write_b32 a6, v6
; GFX908-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX908-NEXT: global_load_dwordx4 v[4:7], v40, s[34:35] offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a8, v8
; GFX908-NEXT: v_accvgpr_write_b32 a9, v9
; GFX908-NEXT: v_accvgpr_write_b32 a10, v10
; GFX908-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX908-NEXT: s_waitcnt vmcnt(4)
; GFX908-NEXT: v_accvgpr_write_b32 a12, v12
; GFX908-NEXT: v_accvgpr_write_b32 a13, v13
; GFX908-NEXT: v_accvgpr_write_b32 a14, v14
; GFX908-NEXT: v_accvgpr_write_b32 a15, v15
+; GFX908-NEXT: s_waitcnt vmcnt(3)
; GFX908-NEXT: v_accvgpr_write_b32 a16, v16
; GFX908-NEXT: v_accvgpr_write_b32 a17, v17
; GFX908-NEXT: v_accvgpr_write_b32 a18, v18
; GFX908-NEXT: v_accvgpr_write_b32 a19, v19
+; GFX908-NEXT: s_waitcnt vmcnt(2)
; GFX908-NEXT: v_accvgpr_write_b32 a20, v20
; GFX908-NEXT: v_accvgpr_write_b32 a21, v21
; GFX908-NEXT: v_accvgpr_write_b32 a22, v22
; GFX908-NEXT: v_accvgpr_write_b32 a23, v23
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v24
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v31
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v7
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v25, v24, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a27
@@ -713,23 +738,22 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v22, a14
; GFX908-NEXT: v_accvgpr_read_b32 v21, a13
; GFX908-NEXT: v_accvgpr_read_b32 v20, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v25, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v24, a0
; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v1, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v0, a4
; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v1, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a4
; GFX908-NEXT: global_store_dwordx4 v40, v[8:11], s[34:35] offset:64
; GFX908-NEXT: global_store_dwordx4 v40, v[12:15], s[34:35] offset:80
; GFX908-NEXT: global_store_dwordx4 v40, v[16:19], s[34:35] offset:32
; GFX908-NEXT: global_store_dwordx4 v40, v[20:23], s[34:35] offset:48
-; GFX908-NEXT: global_store_dwordx4 v40, v[24:27], s[34:35]
-; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:16
+; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35]
+; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:16
; GFX908-NEXT: s_endpgm
bb:
call void @foo()
@@ -928,51 +952,58 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0
; GFX908-LABEL: test_mfma_f32_32x32x1f32_nonentry_noagpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx4 v[30:33], v[0:1], off offset:112
-; GFX908-NEXT: global_load_dwordx4 v[26:29], v[0:1], off offset:96
-; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80
-; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64
-; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48
-; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32
-; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
+; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32
+; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48
+; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64
+; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80
+; GFX908-NEXT: v_mov_b32_e32 v27, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v26, 2.0
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a0, v2
; GFX908-NEXT: v_accvgpr_write_b32 a1, v3
; GFX908-NEXT: v_accvgpr_write_b32 a2, v4
; GFX908-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a4, v6
; GFX908-NEXT: v_accvgpr_write_b32 a5, v7
; GFX908-NEXT: v_accvgpr_write_b32 a6, v8
; GFX908-NEXT: v_accvgpr_write_b32 a7, v9
+; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a8, v10
; GFX908-NEXT: v_accvgpr_write_b32 a9, v11
; GFX908-NEXT: v_accvgpr_write_b32 a10, v12
; GFX908-NEXT: v_accvgpr_write_b32 a11, v13
+; GFX908-NEXT: s_waitcnt vmcnt(4)
; GFX908-NEXT: v_accvgpr_write_b32 a12, v14
; GFX908-NEXT: v_accvgpr_write_b32 a13, v15
; GFX908-NEXT: v_accvgpr_write_b32 a14, v16
; GFX908-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-NEXT: s_waitcnt vmcnt(3)
; GFX908-NEXT: v_accvgpr_write_b32 a16, v18
; GFX908-NEXT: v_accvgpr_write_b32 a17, v19
; GFX908-NEXT: v_accvgpr_write_b32 a18, v20
; GFX908-NEXT: v_accvgpr_write_b32 a19, v21
+; GFX908-NEXT: s_waitcnt vmcnt(2)
; GFX908-NEXT: v_accvgpr_write_b32 a20, v22
; GFX908-NEXT: v_accvgpr_write_b32 a21, v23
; GFX908-NEXT: v_accvgpr_write_b32 a22, v24
; GFX908-NEXT: v_accvgpr_write_b32 a23, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v31
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v32
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v33
-; GFX908-NEXT: v_mov_b32_e32 v2, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v3, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v5
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v8
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v9
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v27, v26, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
@@ -999,23 +1030,22 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0
; GFX908-NEXT: v_accvgpr_read_b32 v24, a14
; GFX908-NEXT: v_accvgpr_read_b32 v23, a13
; GFX908-NEXT: v_accvgpr_read_b32 v22, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a4
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
bb:
@@ -1029,51 +1059,58 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg)
; GFX908-LABEL: test_mfma_f32_32x32x1f32_nonentry_with_agpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx4 v[30:33], v[0:1], off offset:112
-; GFX908-NEXT: global_load_dwordx4 v[26:29], v[0:1], off offset:96
-; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80
-; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64
-; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48
-; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32
-; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
+; GFX908-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:32
+; GFX908-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:48
+; GFX908-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:64
+; GFX908-NEXT: global_load_dwordx4 v[22:25], v[0:1], off offset:80
+; GFX908-NEXT: v_mov_b32_e32 v27, 1.0
+; GFX908-NEXT: v_mov_b32_e32 v26, 2.0
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a0, v2
; GFX908-NEXT: v_accvgpr_write_b32 a1, v3
; GFX908-NEXT: v_accvgpr_write_b32 a2, v4
; GFX908-NEXT: v_accvgpr_write_b32 a3, v5
+; GFX908-NEXT: global_load_dwordx4 v[2:5], v[0:1], off offset:96
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a4, v6
; GFX908-NEXT: v_accvgpr_write_b32 a5, v7
; GFX908-NEXT: v_accvgpr_write_b32 a6, v8
; GFX908-NEXT: v_accvgpr_write_b32 a7, v9
+; GFX908-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:112
+; GFX908-NEXT: s_waitcnt vmcnt(5)
; GFX908-NEXT: v_accvgpr_write_b32 a8, v10
; GFX908-NEXT: v_accvgpr_write_b32 a9, v11
; GFX908-NEXT: v_accvgpr_write_b32 a10, v12
; GFX908-NEXT: v_accvgpr_write_b32 a11, v13
+; GFX908-NEXT: s_waitcnt vmcnt(4)
; GFX908-NEXT: v_accvgpr_write_b32 a12, v14
; GFX908-NEXT: v_accvgpr_write_b32 a13, v15
; GFX908-NEXT: v_accvgpr_write_b32 a14, v16
; GFX908-NEXT: v_accvgpr_write_b32 a15, v17
+; GFX908-NEXT: s_waitcnt vmcnt(3)
; GFX908-NEXT: v_accvgpr_write_b32 a16, v18
; GFX908-NEXT: v_accvgpr_write_b32 a17, v19
; GFX908-NEXT: v_accvgpr_write_b32 a18, v20
; GFX908-NEXT: v_accvgpr_write_b32 a19, v21
+; GFX908-NEXT: s_waitcnt vmcnt(2)
; GFX908-NEXT: v_accvgpr_write_b32 a20, v22
; GFX908-NEXT: v_accvgpr_write_b32 a21, v23
; GFX908-NEXT: v_accvgpr_write_b32 a22, v24
; GFX908-NEXT: v_accvgpr_write_b32 a23, v25
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v26
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v27
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v28
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v29
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v30
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v31
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v32
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v33
-; GFX908-NEXT: v_mov_b32_e32 v2, 1.0
-; GFX908-NEXT: v_mov_b32_e32 v3, 2.0
-; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
+; GFX908-NEXT: s_waitcnt vmcnt(1)
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v5
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v8
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v9
+; GFX908-NEXT: s_nop 0
+; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v27, v26, a[0:31]
; GFX908-NEXT: s_nop 15
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_accvgpr_read_b32 v5, a27
@@ -1100,23 +1137,22 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg)
; GFX908-NEXT: v_accvgpr_read_b32 v24, a14
; GFX908-NEXT: v_accvgpr_read_b32 v23, a13
; GFX908-NEXT: v_accvgpr_read_b32 v22, a12
-; GFX908-NEXT: v_accvgpr_read_b32 v29, a3
-; GFX908-NEXT: v_accvgpr_read_b32 v28, a2
-; GFX908-NEXT: v_accvgpr_read_b32 v27, a1
-; GFX908-NEXT: v_accvgpr_read_b32 v26, a0
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96
-; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: v_accvgpr_read_b32 v5, a7
-; GFX908-NEXT: v_accvgpr_read_b32 v4, a6
-; GFX908-NEXT: v_accvgpr_read_b32 v3, a5
-; GFX908-NEXT: v_accvgpr_read_b32 v2, a4
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112
+; GFX908-NEXT: v_accvgpr_read_b32 v5, a3
+; GFX908-NEXT: v_accvgpr_read_b32 v4, a2
+; GFX908-NEXT: v_accvgpr_read_b32 v3, a1
+; GFX908-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX908-NEXT: v_accvgpr_read_b32 v9, a7
+; GFX908-NEXT: v_accvgpr_read_b32 v8, a6
+; GFX908-NEXT: v_accvgpr_read_b32 v7, a5
+; GFX908-NEXT: v_accvgpr_read_b32 v6, a4
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32
; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off
-; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:16
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
index 60f77bda6d50e..7ec897fc19713 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw-system.ll
@@ -28,29 +28,28 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, ptr addrspace(1
; GCN-NEXT: ; %bb.1: ; %atomic
; GCN-NEXT: s_mov_b32 s8, s10
; GCN-NEXT: s_mov_b32 s9, s10
-; GCN-NEXT: buffer_load_dword v4, v[1:2], s[8:11], 0 addr64 offset:400
+; GCN-NEXT: buffer_load_dword v5, v[1:2], s[8:11], 0 addr64 offset:400
; GCN-NEXT: s_load_dword s4, s[4:5], 0xf
; GCN-NEXT: s_mov_b64 s[2:3], 0
; GCN-NEXT: .LBB0_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_max_i32_e32 v3, s4, v4
-; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v6, v4
-; GCN-NEXT: v_mov_b32_e32 v5, v3
-; GCN-NEXT: buffer_atomic_cmpswap v[5:6], v[1:2], s[8:11], 0 addr64 offset:400 glc
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_max_i32_e32 v4, s4, v5
+; GCN-NEXT: v_mov_b32_e32 v3, v4
+; GCN-NEXT: v_mov_b32_e32 v4, v5
+; GCN-NEXT: buffer_atomic_cmpswap v[3:4], v[1:2], s[8:11], 0 addr64 offset:400 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GCN-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GCN-NEXT: v_mov_b32_e32 v4, v5
+; GCN-NEXT: v_mov_b32_e32 v5, v3
; GCN-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; %atomicrmw.end
; GCN-NEXT: s_or_b64 exec, exec, s[2:3]
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: buffer_store_dword v5, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v3, off, s[0:3], 0
; GCN-NEXT: .LBB0_4: ; %exit
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index d26efd3dc2aa7..1148f65f4460b 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -3003,37 +3003,37 @@ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a,
;
; VI-LABEL: s_mul_i128:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x4c
-; VI-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x7c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4c
+; VI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x7c
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s10
-; VI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], s12, v0, 0
-; VI-NEXT: s_mul_i32 s4, s12, s11
-; VI-NEXT: v_mov_b32_e32 v5, s12
-; VI-NEXT: s_mul_i32 s6, s13, s10
-; VI-NEXT: v_add_u32_e32 v2, vcc, s4, v4
-; VI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s8, v5, 0
-; VI-NEXT: v_add_u32_e32 v4, vcc, s6, v2
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mad_u64_u32 v[3:4], s[12:13], s8, v0, 0
+; VI-NEXT: s_mul_i32 s3, s8, s3
+; VI-NEXT: v_mov_b32_e32 v5, s8
+; VI-NEXT: s_mul_i32 s12, s9, s2
+; VI-NEXT: v_add_u32_e32 v2, vcc, s3, v4
+; VI-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s0, v5, 0
+; VI-NEXT: v_add_u32_e32 v4, vcc, s12, v2
; VI-NEXT: v_mov_b32_e32 v2, 0
-; VI-NEXT: v_mad_u64_u32 v[5:6], s[4:5], s9, v5, v[1:2]
-; VI-NEXT: v_mov_b32_e32 v1, s8
-; VI-NEXT: v_mad_u64_u32 v[3:4], s[4:5], s14, v1, v[3:4]
-; VI-NEXT: v_mov_b32_e32 v7, s13
+; VI-NEXT: v_mad_u64_u32 v[5:6], s[2:3], s1, v5, v[1:2]
+; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s10, v1, v[3:4]
+; VI-NEXT: v_mov_b32_e32 v7, s9
; VI-NEXT: v_mov_b32_e32 v1, v5
-; VI-NEXT: v_mad_u64_u32 v[1:2], s[4:5], s8, v7, v[1:2]
-; VI-NEXT: s_mul_i32 s6, s15, s8
-; VI-NEXT: v_add_u32_e32 v8, vcc, s6, v4
+; VI-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s0, v7, v[1:2]
+; VI-NEXT: s_mul_i32 s8, s11, s0
+; VI-NEXT: v_add_u32_e32 v8, vcc, s8, v4
; VI-NEXT: v_add_u32_e32 v4, vcc, v6, v2
-; VI-NEXT: v_addc_u32_e64 v5, s[4:5], 0, 0, vcc
-; VI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s9, v7, v[4:5]
-; VI-NEXT: s_mul_i32 s6, s14, s9
-; VI-NEXT: v_add_u32_e32 v6, vcc, s6, v8
+; VI-NEXT: v_addc_u32_e64 v5, s[2:3], 0, 0, vcc
+; VI-NEXT: s_mul_i32 s8, s10, s1
+; VI-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s1, v7, v[4:5]
+; VI-NEXT: v_add_u32_e32 v6, vcc, s8, v8
; VI-NEXT: v_add_u32_e32 v2, vcc, v4, v3
; VI-NEXT: v_addc_u32_e32 v3, vcc, v5, v6, vcc
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_mul_i128:
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 10d4eb029ee35..4c641faceb858 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -800,17 +800,18 @@ bb7:
define amdgpu_kernel void @test_umul_i24(ptr addrspace(1) %out, i32 %arg) {
; SI-LABEL: test_umul_i24:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dword s1, s[4:5], 0xb
; SI-NEXT: v_mov_b32_e32 v0, 0xff803fe1
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s2, s2, 9
-; SI-NEXT: s_mul_i32 s4, s2, 0xff803fe1
-; SI-NEXT: v_mul_hi_u32 v1, s2, v0
-; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: s_lshr_b32 s1, s1, 9
+; SI-NEXT: s_mul_i32 s2, s1, 0xff803fe1
+; SI-NEXT: v_mul_hi_u32 v1, s1, v0
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 1
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -823,23 +824,25 @@ define amdgpu_kernel void @test_umul_i24(ptr addrspace(1) %out, i32 %arg) {
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshr_b32 s0, s0, 9
; VI-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s0, v0, 0
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: s_mov_b32 s0, 0
+; VI-NEXT: s_mov_b32 s1, s0
; VI-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
-; VI-NEXT: s_nop 2
+; VI-NEXT: s_nop 1
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_umul_i24:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX9-NEXT: s_mov_b32 s0, 0
; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s2, s2, 9
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, 0xff803fe1
-; GFX9-NEXT: s_mul_i32 s4, s2, 0xff803fe1
+; GFX9-NEXT: s_lshr_b32 s1, s1, 9
+; GFX9-NEXT: s_mul_hi_u32 s5, s1, 0xff803fe1
+; GFX9-NEXT: s_mul_i32 s4, s1, 0xff803fe1
; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_mov_b32 s1, s0
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 9cf59be504cc5..abbb4bf5c5756 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -259,11 +259,11 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:16
; PACKED-SDAG-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1]
; PACKED-SDAG-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:48
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:32
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:80
; PACKED-SDAG-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:64
; PACKED-SDAG-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:112
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:96
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(7)
; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[40:41]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], s[42:43]
@@ -273,23 +273,23 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: v_pk_add_f32 v[8:9], v[8:9], s[48:49]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[10:11], v[10:11], s[50:51]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(4)
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[16:17], v[16:17], s[44:45]
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[18:19], v[18:19], s[46:47]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[20:21], v[20:21], s[44:45]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[22:23], v[22:23], s[46:47]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[28:29], v[28:29], s[16:17]
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[30:31], v[30:31], s[18:19]
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[20:21], v[20:21], s[12:13]
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[22:23], v[22:23], s[14:15]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[16:17], v[16:17], s[16:17]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[18:19], v[18:19], s[18:19]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[28:29], v[28:29], s[12:13]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[30:31], v[30:31], s[14:15]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[14:15], v[14:15], s[10:11]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[24:25], v[24:25], s[20:21]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[26:27], v[26:27], s[22:23]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[4:5], v[4:5], s[36:37]
; PACKED-SDAG-NEXT: v_pk_add_f32 v[12:13], v[12:13], s[8:9]
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:96
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:112
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:64
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:32
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:48
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1]
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
@@ -357,14 +357,14 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_dual_lshlrev_b32 v56, 7, v0 :: v_dual_mov_b32 v32, s40
; GFX1250-SDAG-NEXT: s_clause 0x7
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:16
-; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:80
-; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:96
-; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:64
-; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:112
+; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:96
+; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:64
+; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1] offset:112
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v33, s41 :: v_dual_mov_b32 v34, s42
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v35, s43 :: v_dual_mov_b32 v36, s38
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v39, s49 :: v_dual_mov_b32 v40, s50
@@ -378,38 +378,38 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v43, s45
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v44, s46
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x7
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[32:33]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[24:25], v[24:25], v[32:33]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[26:27], v[26:27], v[34:35]
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v32, s16 :: v_dual_mov_b32 v33, s17
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v34, s18 :: v_dual_mov_b32 v35, s19
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[40:41]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[30:31], v[30:31], v[40:41]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[28:29], v[28:29], v[38:39]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[36:37]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[20:21], v[20:21], v[32:33]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[22:23], v[22:23], v[34:35]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[16:17], v[16:17], v[32:33]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[18:19], v[18:19], v[34:35]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[28:29], v[28:29], v[52:53]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[30:31], v[30:31], v[54:55]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[26:27], v[26:27], v[50:51]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[24:25], v[24:25], v[40:41]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[16:17], v[16:17], v[46:47]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[18:19], v[18:19], v[48:49]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[42:43]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[44:45]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[14:15], v[14:15], v[36:37]
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[52:53]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[14:15], v[14:15], v[54:55]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[50:51]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[40:41]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[46:47]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[48:49]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[20:21], v[20:21], v[42:43]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[22:23], v[22:23], v[44:45]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[36:37]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[38:39]
; GFX1250-SDAG-NEXT: s_clause 0x7
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[20:23], s[0:1] offset:96
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[28:31], s[0:1] offset:112
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[24:27], s[0:1] offset:64
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:80
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:96
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1] offset:112
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:64
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[20:23], s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[28:31], s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[24:27], s[0:1] offset:16
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: fadd_v32_vs:
@@ -1546,11 +1546,11 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:16
; PACKED-SDAG-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1]
; PACKED-SDAG-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:48
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:32
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:80
; PACKED-SDAG-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:64
; PACKED-SDAG-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:112
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:96
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(7)
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], s[40:41]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[2:3], v[2:3], s[42:43]
@@ -1560,23 +1560,23 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[8:9], v[8:9], s[48:49]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[10:11], v[10:11], s[50:51]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(4)
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[16:17], v[16:17], s[44:45]
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[18:19], v[18:19], s[46:47]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[20:21], v[20:21], s[44:45]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[22:23], v[22:23], s[46:47]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[28:29], v[28:29], s[16:17]
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[30:31], v[30:31], s[18:19]
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[20:21], v[20:21], s[12:13]
-; PACKED-SDAG-NEXT: v_pk_mul_f32 v[22:23], v[22:23], s[14:15]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[16:17], v[16:17], s[16:17]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[18:19], v[18:19], s[18:19]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[28:29], v[28:29], s[12:13]
+; PACKED-SDAG-NEXT: v_pk_mul_f32 v[30:31], v[30:31], s[14:15]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[14:15], v[14:15], s[10:11]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[24:25], v[24:25], s[20:21]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[26:27], v[26:27], s[22:23]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[4:5], v[4:5], s[36:37]
; PACKED-SDAG-NEXT: v_pk_mul_f32 v[12:13], v[12:13], s[8:9]
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:96
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:112
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:64
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:32
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:48
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1]
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
@@ -1644,14 +1644,14 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_dual_lshlrev_b32 v56, 7, v0 :: v_dual_mov_b32 v32, s40
; GFX1250-SDAG-NEXT: s_clause 0x7
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:16
-; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:80
-; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:96
-; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:64
-; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:112
+; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:96
+; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:64
+; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1] offset:112
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v33, s41 :: v_dual_mov_b32 v34, s42
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v35, s43 :: v_dual_mov_b32 v36, s38
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v39, s49 :: v_dual_mov_b32 v40, s50
@@ -1665,38 +1665,38 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v43, s45
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v44, s46
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x7
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[32:33]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[32:33]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[34:35]
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v32, s16 :: v_dual_mov_b32 v33, s17
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v34, s18 :: v_dual_mov_b32 v35, s19
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[40:41]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[30:31], v[30:31], v[40:41]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[28:29], v[28:29], v[38:39]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[36:37]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[32:33]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[34:35]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[32:33]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[34:35]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[28:29], v[28:29], v[52:53]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[30:31], v[30:31], v[54:55]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[26:27], v[26:27], v[50:51]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[24:25], v[24:25], v[40:41]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[16:17], v[16:17], v[46:47]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[18:19], v[18:19], v[48:49]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[42:43]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[44:45]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[36:37]
-; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[12:13], v[12:13], v[52:53]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[14:15], v[14:15], v[54:55]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[10:11], v[10:11], v[50:51]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[8:9], v[8:9], v[40:41]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[4:5], v[4:5], v[46:47]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[6:7], v[6:7], v[48:49]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[20:21], v[20:21], v[42:43]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[22:23], v[22:23], v[44:45]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[36:37]
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[38:39]
; GFX1250-SDAG-NEXT: s_clause 0x7
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[20:23], s[0:1] offset:96
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[28:31], s[0:1] offset:112
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[24:27], s[0:1] offset:64
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:80
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:96
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1] offset:112
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:64
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[20:23], s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[28:31], s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[24:27], s[0:1] offset:16
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: fmul_v32_vs:
@@ -2394,11 +2394,11 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:16
; PACKED-SDAG-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1]
; PACKED-SDAG-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:48
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:32
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:80
; PACKED-SDAG-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:64
; PACKED-SDAG-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:112
-; PACKED-SDAG-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:96
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(7)
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[0:1], v[0:1], s[40:41], s[40:41]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[2:3], v[2:3], s[42:43], s[42:43]
@@ -2408,23 +2408,23 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[8:9], v[8:9], s[48:49], s[48:49]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[10:11], v[10:11], s[50:51], s[50:51]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(4)
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[16:17], v[16:17], s[44:45], s[44:45]
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[18:19], v[18:19], s[46:47], s[46:47]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[20:21], v[20:21], s[44:45], s[44:45]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[22:23], v[22:23], s[46:47], s[46:47]
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[28:29], v[28:29], s[16:17], s[16:17]
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[30:31], v[30:31], s[18:19], s[18:19]
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[20:21], v[20:21], s[12:13], s[12:13]
-; PACKED-SDAG-NEXT: v_pk_fma_f32 v[22:23], v[22:23], s[14:15], s[14:15]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[16:17], v[16:17], s[16:17], s[16:17]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[18:19], v[18:19], s[18:19], s[18:19]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[28:29], v[28:29], s[12:13], s[12:13]
+; PACKED-SDAG-NEXT: v_pk_fma_f32 v[30:31], v[30:31], s[14:15], s[14:15]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[14:15], v[14:15], s[10:11], s[10:11]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[24:25], v[24:25], s[20:21], s[20:21]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[26:27], v[26:27], s[22:23], s[22:23]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[4:5], v[4:5], s[36:37], s[36:37]
; PACKED-SDAG-NEXT: v_pk_fma_f32 v[12:13], v[12:13], s[8:9], s[8:9]
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:96
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:96
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:112
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:64
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:80
+; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:32
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:48
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1]
; PACKED-SDAG-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16
@@ -2492,11 +2492,11 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: v_lshlrev_b32_e32 v56, 7, v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_clause 0x7
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:16
-; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1]
-; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:80
; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:96
; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:64
; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:112
@@ -2513,14 +2513,14 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[44:45], s[46:47]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[36:37], s[38:39]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x7
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[32:33], v[32:33]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[2:3], v[2:3], v[34:35], v[34:35]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[8:9], v[8:9], v[32:33], v[32:33]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[10:11], v[10:11], v[34:35], v[34:35]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[32:33], s[16:17]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[34:35], s[18:19]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[6:7], v[6:7], v[40:41], v[40:41]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[14:15], v[14:15], v[40:41], v[40:41]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[4:5], v[4:5], v[38:39], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[12:13], v[12:13], v[38:39], v[38:39]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[38:39], s[36:37]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x2
; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[20:21], v[20:21], v[32:33], v[32:33]
@@ -2530,21 +2530,21 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[30:31], v[30:31], v[54:55], v[54:55]
; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[26:27], v[26:27], v[50:51], v[50:51]
; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[24:25], v[24:25], v[40:41], v[40:41]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[16:17], v[16:17], v[46:47], v[46:47]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[18:19], v[18:19], v[48:49], v[48:49]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[8:9], v[8:9], v[42:43], v[42:43]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[10:11], v[10:11], v[44:45], v[44:45]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[14:15], v[14:15], v[36:37], v[36:37]
-; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[12:13], v[12:13], v[38:39], v[38:39]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[4:5], v[4:5], v[46:47], v[46:47]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[6:7], v[6:7], v[48:49], v[48:49]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[16:17], v[16:17], v[42:43], v[42:43]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[18:19], v[18:19], v[44:45], v[44:45]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[2:3], v[2:3], v[36:37], v[36:37]
+; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[38:39], v[38:39]
; GFX1250-SDAG-NEXT: s_clause 0x7
; GFX1250-SDAG-NEXT: global_store_b128 v56, v[20:23], s[0:1] offset:96
; GFX1250-SDAG-NEXT: global_store_b128 v56, v[28:31], s[0:1] offset:112
; GFX1250-SDAG-NEXT: global_store_b128 v56, v[24:27], s[0:1] offset:64
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:80
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:32
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:48
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1]
-; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:80
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[16:19], s[0:1] offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[12:15], s[0:1] offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v56, v[8:11], s[0:1] offset:16
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: fma_v32_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-reg-sequence-copy-folding.ll b/llvm/test/CodeGen/AMDGPU/peephole-opt-reg-sequence-copy-folding.ll
new file mode 100644
index 0000000000000..90310953a6252
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-reg-sequence-copy-folding.ll
@@ -0,0 +1,275 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -pass-remarks-analysis=kernel-resource-usage < %s 2>&1 | FileCheck %s
+;
+; Show reduction in VGPR spilling from avoiding to use subregister sources in the coalescable copy
+; optimization for REG_SQUENCE instructions.
+
+; CHECK: ScratchSize [bytes/lane]: 24{{$}}
+; CHECK: VGPRs Spill: 5{{$}}
+
+
+define amdgpu_kernel void @snork(ptr %arg4, ptr addrspace(3) %arg21, ptr addrspace(3) %arg26, ptr addrspace(3) %arg30, ptr addrspace(1) %arg33, i32 %arg34, i32 %arg35, <8 x i32> %arg37, i32 %arg38, i32 %arg39, i32 %arg40, i32 %arg41, <2 x i128> %arg42, i32 %arg43, i32 %arg44, <2 x i128> %arg46, i32 %arg47, i32 %arg48, i32 %arg49, i32 %arg50, i1 %arg52) #0 {
+bb:
+ %call = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %lshr = lshr i32 %call, 1
+ %shl = shl nuw nsw i32 %call, 10
+ %and = and i32 %shl, 1040384
+ %shl61 = shl nuw nsw i32 %call, 1
+ %zext62 = zext nneg i32 %shl61 to i64
+ %getelementptr74 = getelementptr inbounds nuw i8, ptr %arg4, i64 %zext62
+ %ptrtoint75 = ptrtoint ptr %getelementptr74 to i64
+ %trunc76 = trunc i64 %ptrtoint75 to i32
+ %add77 = add i32 %trunc76, 1
+ %getelementptr82 = getelementptr inbounds nuw i8, ptr addrspace(3) null, i32 %lshr
+ %addrspacecast = addrspacecast ptr addrspace(3) %getelementptr82 to ptr
+ %ptrtoint83 = ptrtoint ptr %addrspacecast to i64
+ %trunc84 = trunc i64 %ptrtoint83 to i32
+ %inttoptr = inttoptr i32 %trunc84 to ptr addrspace(3)
+ %getelementptr85 = getelementptr inbounds nuw i8, ptr %addrspacecast, i64 8192
+ %ptrtoint86 = ptrtoint ptr %getelementptr85 to i64
+ %trunc87 = trunc i64 %ptrtoint86 to i32
+ %inttoptr88 = inttoptr i32 %trunc87 to ptr addrspace(3)
+ %getelementptr99 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg21, i32 %lshr
+ %addrspacecast100 = addrspacecast ptr addrspace(3) %getelementptr99 to ptr
+ %ptrtoint101 = ptrtoint ptr %addrspacecast100 to i64
+ %trunc102 = trunc i64 %ptrtoint101 to i32
+ %inttoptr103 = inttoptr i32 %trunc102 to ptr addrspace(3)
+ %getelementptr104 = getelementptr inbounds nuw i8, ptr %addrspacecast100, i64 8192
+ %ptrtoint105 = ptrtoint ptr %getelementptr104 to i64
+ %trunc106 = trunc i64 %ptrtoint105 to i32
+ %inttoptr107 = inttoptr i32 %trunc106 to ptr addrspace(3)
+ %getelementptr113 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg26, i32 %lshr
+ %addrspacecast114 = addrspacecast ptr addrspace(3) %getelementptr113 to ptr
+ %ptrtoint115 = ptrtoint ptr %addrspacecast114 to i64
+ %trunc116 = trunc i64 %ptrtoint115 to i32
+ %inttoptr117 = inttoptr i32 %trunc116 to ptr addrspace(3)
+ %getelementptr118 = getelementptr inbounds nuw i8, ptr %addrspacecast114, i64 8192
+ %ptrtoint119 = ptrtoint ptr %getelementptr118 to i64
+ %trunc120 = trunc i64 %ptrtoint119 to i32
+ %inttoptr121 = inttoptr i32 %trunc120 to ptr addrspace(3)
+ %getelementptr127 = getelementptr inbounds nuw i8, ptr addrspace(3) %arg30, i32 %lshr
+ %addrspacecast128 = addrspacecast ptr addrspace(3) %getelementptr127 to ptr
+ %ptrtoint129 = ptrtoint ptr %addrspacecast128 to i64
+ %trunc130 = trunc i64 %ptrtoint129 to i32
+ %inttoptr131 = inttoptr i32 %trunc130 to ptr addrspace(3)
+ %getelementptr132 = getelementptr inbounds nuw i8, ptr %addrspacecast128, i64 8192
+ %ptrtoint133 = ptrtoint ptr %getelementptr132 to i64
+ %trunc134 = trunc i64 %ptrtoint133 to i32
+ %inttoptr135 = inttoptr i32 %trunc134 to ptr addrspace(3)
+ br label %bb166
+
+bb136: ; preds = %bb166
+ %and57 = and i32 %call, 48
+ %and141 = shl nuw nsw i32 %call, 1
+ %shl142 = and i32 %and141, 2
+ %or143 = or disjoint i32 %shl142, %and57
+ %call148 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %call151 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %call153 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %add154 = add i32 %or143, %trunc76
+ %call155 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %add154, i32 0) #6
+ %call157 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %call161 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ store i16 0, ptr addrspace(1) %arg33, align 2
+ %call165 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ ret void
+
+bb166: ; preds = %bb166, %bb
+ %phi167 = phi <4 x float> [ zeroinitializer, %bb ], [ %call312, %bb166 ]
+ %phi168 = phi <4 x float> [ zeroinitializer, %bb ], [ %call313, %bb166 ]
+ %phi169 = phi <4 x float> [ zeroinitializer, %bb ], [ %call314, %bb166 ]
+ %phi170 = phi <4 x float> [ zeroinitializer, %bb ], [ %call315, %bb166 ]
+ %phi171 = phi <4 x float> [ zeroinitializer, %bb ], [ %call316, %bb166 ]
+ %phi172 = phi <4 x float> [ zeroinitializer, %bb ], [ %call317, %bb166 ]
+ %phi173 = phi <4 x float> [ zeroinitializer, %bb ], [ %call318, %bb166 ]
+ %phi174 = phi <4 x float> [ zeroinitializer, %bb ], [ %call304, %bb166 ]
+ %phi175 = phi <4 x float> [ zeroinitializer, %bb ], [ %call305, %bb166 ]
+ %phi176 = phi <4 x float> [ zeroinitializer, %bb ], [ %call307, %bb166 ]
+ %phi177 = phi <4 x float> [ zeroinitializer, %bb ], [ %call308, %bb166 ]
+ %phi178 = phi <4 x float> [ zeroinitializer, %bb ], [ %call309, %bb166 ]
+ %phi179 = phi <4 x float> [ zeroinitializer, %bb ], [ %call310, %bb166 ]
+ %phi180 = phi <4 x float> [ zeroinitializer, %bb ], [ %call311, %bb166 ]
+ %phi181 = phi <4 x float> [ zeroinitializer, %bb ], [ %call289, %bb166 ]
+ %phi182 = phi <4 x float> [ zeroinitializer, %bb ], [ %call290, %bb166 ]
+ %phi183 = phi <4 x float> [ zeroinitializer, %bb ], [ %call291, %bb166 ]
+ %phi184 = phi <4 x float> [ zeroinitializer, %bb ], [ %call292, %bb166 ]
+ %phi185 = phi <4 x float> [ zeroinitializer, %bb ], [ %call293, %bb166 ]
+ %phi186 = phi <4 x float> [ zeroinitializer, %bb ], [ %call294, %bb166 ]
+ %phi187 = phi <4 x float> [ zeroinitializer, %bb ], [ %call295, %bb166 ]
+ %phi188 = phi <4 x float> [ zeroinitializer, %bb ], [ %call277, %bb166 ]
+ %phi189 = phi <4 x float> [ zeroinitializer, %bb ], [ %call278, %bb166 ]
+ %phi190 = phi <4 x float> [ zeroinitializer, %bb ], [ %call280, %bb166 ]
+ %phi191 = phi <4 x float> [ zeroinitializer, %bb ], [ %call281, %bb166 ]
+ %phi192 = phi <4 x float> [ zeroinitializer, %bb ], [ %call283, %bb166 ]
+ %phi193 = phi <4 x float> [ zeroinitializer, %bb ], [ %call284, %bb166 ]
+ %call194 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg34, i32 0) #6
+ %call195 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call196 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %call197 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call198 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg35, i32 0) #6
+ %call199 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call200 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call201 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement202 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call201, i64 0
+ %call203 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement204 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call203, i64 0
+ %call205 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi188, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call206 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %arg37, <4 x float> %phi189, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast207 = bitcast <2 x i128> %insertelement202 to <8 x i32>
+ %call208 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast207, <8 x i32> zeroinitializer, <4 x float> %phi190, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call209 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi191, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast210 = bitcast <2 x i128> %insertelement204 to <8 x i32>
+ %call211 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast210, <8 x i32> zeroinitializer, <4 x float> %phi192, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call212 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi193, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call213 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg38, i32 0) #6
+ %insertelement214 = insertelement <2 x i128> poison, i128 %call213, i64 0
+ %call215 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call216 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg39, i32 0) #6
+ %insertelement217 = insertelement <2 x i128> %insertelement214, i128 %call216, i64 1
+ %call218 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement219 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call218, i64 0
+ %bitcast220 = bitcast <2 x i128> %insertelement217 to <8 x i32>
+ %bitcast221 = bitcast <2 x i128> %insertelement219 to <8 x i32>
+ %call222 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %bitcast221, <4 x float> %phi181, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call223 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi182, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call224 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi183, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call225 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast207, <8 x i32> zeroinitializer, <4 x float> %phi184, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call226 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi185, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call227 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi186, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call228 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast210, <8 x i32> zeroinitializer, <4 x float> %phi187, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call229 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %add77, i32 0) #6
+ %call230 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call231 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement232 = insertelement <2 x i128> <i128 0, i128 poison>, i128 %call231, i64 1
+ %call233 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement234 = insertelement <2 x i128> <i128 0, i128 poison>, i128 %call233, i64 1
+ %call235 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg40, i32 0) #6
+ %insertelement236 = insertelement <2 x i128> <i128 0, i128 poison>, i128 %call235, i64 1
+ %call237 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement238 = insertelement <2 x i128> <i128 0, i128 poison>, i128 %call237, i64 1
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr, i32 noundef 1, i32 noundef %and, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr88, i32 noundef 1, i32 noundef 524288, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.s.barrier()
+ tail call void asm sideeffect "s_waitcnt lgkmcnt(0)", ""() #6
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %bitcast239 = bitcast <2 x i128> %insertelement232 to <8 x i32>
+ %call240 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> splat (i32 1), <8 x i32> zeroinitializer, <4 x float> %phi174, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call241 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi175, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast242 = bitcast <2 x i128> %insertelement234 to <8 x i32>
+ %call243 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast242, <8 x i32> zeroinitializer, <4 x float> %phi176, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call244 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %arg37, <4 x float> %phi177, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast245 = bitcast <2 x i128> %insertelement236 to <8 x i32>
+ %call246 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast245, <8 x i32> zeroinitializer, <4 x float> %phi178, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast247 = bitcast <2 x i128> %insertelement238 to <8 x i32>
+ %call248 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast247, <8 x i32> zeroinitializer, <4 x float> %phi179, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call249 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi180, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call250 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast239, <8 x i32> %bitcast221, <4 x float> %phi167, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call251 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast242, <8 x i32> %bitcast220, <4 x float> %phi168, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call252 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi169, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call253 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi170, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call254 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi171, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call255 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %phi172, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call256 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast247, <8 x i32> zeroinitializer, <4 x float> %phi173, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %call257 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg41, i32 0) #6
+ %call259 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call261 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg43, i32 0) #6
+ %call263 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg44, i32 0) #6
+ %call264 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 1, i32 0) #6
+ %insertelement265 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call264, i64 0
+ %call266 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call268 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement269 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call268, i64 0
+ %call270 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement271 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call270, i64 0
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr103, i32 noundef 1, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr107, i32 noundef 1, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %bitcast273 = bitcast <2 x i128> %insertelement265 to <8 x i32>
+ %bitcast274 = bitcast <2 x i128> %arg46 to <8 x i32>
+ %bitcast275 = bitcast <2 x i128> %arg42 to <8 x i32>
+ %bitcast276 = bitcast <2 x i128> %arg42 to <8 x i32>
+ %call277 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> splat (i32 1), <8 x i32> %bitcast274, <4 x float> %call205, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call278 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %bitcast275, <4 x float> %call206, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast279 = bitcast <2 x i128> %insertelement269 to <8 x i32>
+ %call280 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call208, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call281 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call209, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast282 = bitcast <2 x i128> %insertelement271 to <8 x i32>
+ %call283 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast282, <8 x i32> zeroinitializer, <4 x float> %call211, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call284 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call212, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %call285 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg47, i32 0) #6
+ %call286 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg48, i32 0) #6
+ %insertelement287 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call286, i64 0
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr117, i32 noundef 1, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr121, i32 noundef 1, i32 noundef 524288, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %bitcast288 = bitcast <2 x i128> %insertelement287 to <8 x i32>
+ %call289 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast273, <8 x i32> zeroinitializer, <4 x float> %call222, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call290 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast276, <8 x i32> %bitcast288, <4 x float> %call223, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call291 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call224, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call292 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call225, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call293 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast279, <8 x i32> zeroinitializer, <4 x float> %call226, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call294 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call227, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call295 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast282, <8 x i32> zeroinitializer, <4 x float> %call228, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ tail call void @llvm.amdgcn.s.barrier()
+ %call296 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg49, i32 0) #6
+ %call297 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %call298 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 %arg50, i32 0) #6
+ %insertelement299 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call298, i64 0
+ %call300 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ %insertelement301 = insertelement <2 x i128> <i128 poison, i128 0>, i128 %call300, i64 0
+ %call302 = tail call i128 asm sideeffect "ds_read_b128 $0, $1 offset:$2\0A", "=v,v,i,~{memory}"(i32 0, i32 0) #6
+ tail call void @llvm.amdgcn.s.barrier()
+ tail call void asm sideeffect "s_waitcnt lgkmcnt(0)", ""() #6
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %bitcast303 = bitcast <2 x i128> %insertelement299 to <8 x i32>
+ %call304 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast303, <8 x i32> zeroinitializer, <4 x float> %call240, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call305 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call241, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %bitcast306 = bitcast <2 x i128> %insertelement301 to <8 x i32>
+ %call307 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> splat (i32 1), <8 x i32> zeroinitializer, <4 x float> %call243, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call308 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call244, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call309 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> splat (i32 1), <4 x float> %call246, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call310 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %bitcast274, <4 x float> %call248, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call311 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call249, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ tail call void @llvm.amdgcn.s.barrier()
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr131, i32 noundef 1, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> noundef zeroinitializer, ptr addrspace(3) noundef %inttoptr135, i32 noundef 1, i32 noundef 0, i32 noundef 0, i32 noundef 0, i32 noundef 0) #6
+ tail call void @llvm.amdgcn.s.setprio(i16 0)
+ %call312 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast303, <8 x i32> zeroinitializer, <4 x float> %call250, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call313 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call251, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call314 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %bitcast306, <8 x i32> zeroinitializer, <4 x float> %call252, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call315 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> %bitcast288, <4 x float> %call253, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call316 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call254, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call317 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call255, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ %call318 = tail call contract <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer, <4 x float> %call256, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+ br i1 %arg52, label %bb136, label %bb166
+}
+
+; Function Attrs: convergent nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.s.barrier() #1
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare void @llvm.amdgcn.s.setprio(i16 immarg) #2
+
+; Function Attrs: convergent nocallback nofree nounwind willreturn
+
+; Function Attrs: nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32>, ptr addrspace(3) captures(none), i32 immarg, i32, i32, i32 immarg, i32 immarg) #3
+
+; Function Attrs: convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
+declare <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32>, <8 x i32>, <4 x float>, i32 immarg, i32 immarg, i32 immarg, i32, i32 immarg, i32) #4
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #5
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+
+
+attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2" }
+attributes #2 = { nocallback nofree nosync nounwind willreturn }
+attributes #3 = { nocallback nofree nounwind willreturn }
+attributes #4 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
+attributes #5 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #6 = { convergent nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
index c9645c31aad75..313fdda7f2379 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
@@ -48,7 +48,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]].sub0_sub1, %subreg.sub2_sub3, [[COPY1]].sub2_sub3, %subreg.sub0_sub1
; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]].sub1_sub2_sub3, %subreg.sub0_sub1_sub2, [[COPY1]].sub0, %subreg.sub3
- ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]].sub1, %subreg.sub0, [[REG_SEQUENCE]].sub2, %subreg.sub1, [[REG_SEQUENCE]].sub3, %subreg.sub2, [[COPY1]].sub0, %subreg.sub3
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]].sub0, %subreg.sub0, [[REG_SEQUENCE1]].sub1, %subreg.sub1, [[REG_SEQUENCE1]].sub2, %subreg.sub2, [[REG_SEQUENCE1]].sub3, %subreg.sub3
; GCN-NEXT: KILL implicit [[REG_SEQUENCE2]]
%0:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -70,8 +70,8 @@ body: |
; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]].sub0_sub1, %subreg.sub2_sub3, [[COPY1]].sub2_sub3, %subreg.sub0_sub1
- ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]].sub2_sub3, %subreg.sub2_sub3, [[COPY]].sub0, %subreg.sub1, [[COPY]].sub1, %subreg.sub0
- ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]].sub1, %subreg.sub0, [[COPY]].sub0, %subreg.sub1, [[COPY1]].sub2, %subreg.sub2, [[COPY1]].sub3, %subreg.sub3
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]].sub0_sub1, %subreg.sub2_sub3, [[REG_SEQUENCE]].sub2, %subreg.sub1, [[REG_SEQUENCE]].sub3, %subreg.sub0
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]].sub0, %subreg.sub0, [[REG_SEQUENCE1]].sub1, %subreg.sub1, [[REG_SEQUENCE1]].sub2, %subreg.sub2, [[REG_SEQUENCE1]].sub3, %subreg.sub3
; GCN-NEXT: KILL implicit [[REG_SEQUENCE2]]
%0:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:vreg_128 = COPY $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 80b5536b7e160..de720acfa4d3b 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -35,81 +35,81 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or_b32_e32 v7, v1, v3
; GFX9-NEXT: v_or_b32_e32 v6, v0, v2
; GFX9-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[6:7]
-; GFX9-NEXT: v_ffbh_u32_e32 v6, v4
-; GFX9-NEXT: v_add_u32_e32 v6, 32, v6
-; GFX9-NEXT: v_ffbh_u32_e32 v7, v5
-; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX9-NEXT: v_ffbh_u32_e32 v7, v23
+; GFX9-NEXT: v_ffbh_u32_e32 v7, v4
; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
-; GFX9-NEXT: v_ffbh_u32_e32 v8, v22
+; GFX9-NEXT: v_ffbh_u32_e32 v8, v5
; GFX9-NEXT: v_min_u32_e32 v7, v7, v8
+; GFX9-NEXT: v_ffbh_u32_e32 v8, v23
+; GFX9-NEXT: v_add_u32_e32 v8, 32, v8
+; GFX9-NEXT: v_ffbh_u32_e32 v9, v22
+; GFX9-NEXT: v_min_u32_e32 v8, v8, v9
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7
-; GFX9-NEXT: v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, 64, v8
+; GFX9-NEXT: v_addc_co_u32_e64 v9, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX9-NEXT: v_ffbh_u32_e32 v10, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX9-NEXT: v_ffbh_u32_e32 v7, v2
-; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
-; GFX9-NEXT: v_min_u32_e32 v7, v7, v10
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX9-NEXT: v_ffbh_u32_e32 v8, v2
+; GFX9-NEXT: v_add_u32_e32 v8, 32, v8
+; GFX9-NEXT: v_min_u32_e32 v8, v8, v10
; GFX9-NEXT: v_ffbh_u32_e32 v10, v0
; GFX9-NEXT: v_add_u32_e32 v10, 32, v10
; GFX9-NEXT: v_ffbh_u32_e32 v11, v1
; GFX9-NEXT: v_min_u32_e32 v10, v10, v11
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, 64, v10
; GFX9-NEXT: v_addc_co_u32_e64 v11, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v9, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc
-; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7
-; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v11, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v9, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v7, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v9, v11, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v6, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v6, vcc
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_or_b32_e32 v13, v9, v11
+; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: v_mov_b32_e32 v21, v20
-; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
-; GFX9-NEXT: v_and_b32_e32 v10, 1, v10
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
-; GFX9-NEXT: v_xor_b32_e32 v10, 0x7f, v6
-; GFX9-NEXT: v_or_b32_e32 v11, v7, v9
-; GFX9-NEXT: v_or_b32_e32 v10, v10, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
+; GFX9-NEXT: v_and_b32_e32 v6, 1, v6
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_xor_b32_e32 v6, 0x7f, v8
+; GFX9-NEXT: v_or_b32_e32 v12, v6, v10
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GFX9-NEXT: v_cndmask_b32_e64 v13, v3, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v12, v2, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v10, v0, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v3, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v2, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v12, v1, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v13, v0, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v6
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v7, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v8, vcc
-; GFX9-NEXT: v_sub_u32_e32 v13, 0x7f, v6
-; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v9, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v8
+; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v9, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v10, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, 0, v11, vcc
+; GFX9-NEXT: v_sub_u32_e32 v13, 0x7f, v8
+; GFX9-NEXT: v_or_b32_e32 v7, v25, v27
+; GFX9-NEXT: v_or_b32_e32 v6, v24, v26
; GFX9-NEXT: v_sub_u32_e32 v11, 64, v13
-; GFX9-NEXT: v_or_b32_e32 v8, v25, v27
-; GFX9-NEXT: v_or_b32_e32 v7, v24, v26
; GFX9-NEXT: v_lshlrev_b64 v[9:10], v13, v[2:3]
; GFX9-NEXT: v_lshrrev_b64 v[11:12], v11, v[0:1]
-; GFX9-NEXT: v_sub_u32_e32 v6, 63, v6
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT: v_sub_u32_e32 v6, 63, v8
; GFX9-NEXT: v_lshlrev_b64 v[6:7], v6, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v8, v10, v12
; GFX9-NEXT: v_or_b32_e32 v9, v9, v11
; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v13
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v10, v10, v12
; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[4:5]
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v13, v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v10, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v13
; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v3, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v2, s[6:7]
@@ -192,34 +192,34 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: .LBB0_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: v_lshlrev_b64 v[12:13], 1, v[6:7]
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[8:9]
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v9
-; GFX9-NEXT: v_or_b32_e32 v12, v12, v14
-; GFX9-NEXT: v_or_b32_e32 v11, v11, v7
-; GFX9-NEXT: v_or_b32_e32 v10, v10, v6
+; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[6:7]
+; GFX9-NEXT: v_lshrrev_b32_e32 v12, 31, v9
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
+; GFX9-NEXT: v_or_b32_e32 v6, v6, v12
+; GFX9-NEXT: v_or_b32_e32 v12, v11, v9
+; GFX9-NEXT: v_or_b32_e32 v13, v10, v8
; GFX9-NEXT: .LBB0_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v23, v10, 0
-; GFX9-NEXT: v_mov_b32_e32 v8, 0
-; GFX9-NEXT: v_mul_lo_u32 v9, v11, v4
-; GFX9-NEXT: v_mul_lo_u32 v16, v10, v5
-; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v22, v10, v[7:8]
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v10, v4, 0
-; GFX9-NEXT: v_mul_lo_u32 v10, v12, v22
-; GFX9-NEXT: v_mul_lo_u32 v13, v13, v23
-; GFX9-NEXT: v_mov_b32_e32 v7, v14
-; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v23, v11, v[7:8]
-; GFX9-NEXT: v_add3_u32 v5, v5, v16, v9
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v12, v23, v[4:5]
-; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v15, v8
-; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v22, v11, v[8:9]
-; GFX9-NEXT: v_add3_u32 v5, v13, v5, v10
-; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v8, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v9, v5, vcc
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v6
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v23, v13, 0
+; GFX9-NEXT: v_mov_b32_e32 v10, 0
+; GFX9-NEXT: v_mul_lo_u32 v11, v12, v4
+; GFX9-NEXT: v_mul_lo_u32 v16, v13, v5
+; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v22, v13, v[9:10]
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v13, v4, 0
+; GFX9-NEXT: v_mul_lo_u32 v13, v6, v22
+; GFX9-NEXT: v_mul_lo_u32 v17, v7, v23
+; GFX9-NEXT: v_mov_b32_e32 v9, v14
+; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v23, v12, v[9:10]
+; GFX9-NEXT: v_add3_u32 v5, v5, v16, v11
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v6, v23, v[4:5]
+; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v15, v10
+; GFX9-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, 0, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v22, v12, v[6:7]
+; GFX9-NEXT: v_add3_u32 v5, v17, v5, v13
+; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v6, v4
+; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v5, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v9, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc
; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20
@@ -1481,68 +1481,68 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX9-NEXT: v_ffbh_u32_e32 v11, v3
+; GFX9-NEXT: v_ffbh_u32_e32 v12, v1
; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
; GFX9-NEXT: v_ffbh_u32_e32 v9, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v10, 0, vcc
; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
-; GFX9-NEXT: v_min_u32_e32 v9, v9, v11
-; GFX9-NEXT: v_ffbh_u32_e32 v11, v0
-; GFX9-NEXT: v_add_u32_e32 v11, 32, v11
-; GFX9-NEXT: v_ffbh_u32_e32 v12, v1
-; GFX9-NEXT: v_min_u32_e32 v11, v11, v12
-; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11
+; GFX9-NEXT: v_ffbh_u32_e32 v10, v3
+; GFX9-NEXT: v_min_u32_e32 v9, v9, v10
+; GFX9-NEXT: v_ffbh_u32_e32 v10, v0
+; GFX9-NEXT: v_add_u32_e32 v10, 32, v10
+; GFX9-NEXT: v_min_u32_e32 v10, v10, v12
+; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, 64, v10
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc
-; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v9
-; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v10, v12, vcc
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
-; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v11, vcc
-; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v11, vcc
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
-; GFX9-NEXT: v_and_b32_e32 v12, 1, v12
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12
-; GFX9-NEXT: v_xor_b32_e32 v12, 0x7f, v8
-; GFX9-NEXT: v_or_b32_e32 v13, v9, v11
-; GFX9-NEXT: v_or_b32_e32 v12, v12, v10
-; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v9
+; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v12, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_subb_co_u32_e32 v12, vcc, 0, v8, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, 0, v8, vcc
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[10:11]
+; GFX9-NEXT: v_or_b32_e32 v15, v11, v13
+; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX9-NEXT: v_and_b32_e32 v8, 1, v8
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
+; GFX9-NEXT: v_xor_b32_e32 v8, 0x7f, v10
+; GFX9-NEXT: v_or_b32_e32 v14, v8, v12
+; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
+; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v2, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v13, v1, 0, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v9, v3, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v8, v2, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v14, v1, 0, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v15, v0, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB1_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
-; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v8
-; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v9, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v10, vcc
-; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v8
-; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v11, vcc
+; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v10
+; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v11, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v12, vcc
+; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v13, vcc
+; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v10
+; GFX9-NEXT: v_or_b32_e32 v9, v23, v25
+; GFX9-NEXT: v_or_b32_e32 v8, v22, v24
; GFX9-NEXT: v_sub_u32_e32 v13, 64, v15
-; GFX9-NEXT: v_or_b32_e32 v10, v23, v25
-; GFX9-NEXT: v_or_b32_e32 v9, v22, v24
; GFX9-NEXT: v_lshlrev_b64 v[11:12], v15, v[2:3]
; GFX9-NEXT: v_lshrrev_b64 v[13:14], v13, v[0:1]
-; GFX9-NEXT: v_sub_u32_e32 v8, 63, v8
-; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[9:10]
+; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT: v_sub_u32_e32 v8, 63, v10
; GFX9-NEXT: v_lshlrev_b64 v[8:9], v8, v[0:1]
-; GFX9-NEXT: v_or_b32_e32 v10, v12, v14
; GFX9-NEXT: v_or_b32_e32 v11, v11, v13
; GFX9-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v15
-; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v10, s[4:5]
+; GFX9-NEXT: v_or_b32_e32 v12, v12, v14
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v11, s[4:5]
; GFX9-NEXT: v_lshlrev_b64 v[10:11], v15, v[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v12, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v15
; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v3, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v2, s[6:7]
@@ -1625,34 +1625,34 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: .LBB1_5: ; %Flow2
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: v_lshlrev_b64 v[14:15], 1, v[8:9]
-; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[10:11]
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 31, v11
-; GFX9-NEXT: v_or_b32_e32 v14, v14, v16
-; GFX9-NEXT: v_or_b32_e32 v13, v13, v9
-; GFX9-NEXT: v_or_b32_e32 v12, v12, v8
+; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[8:9]
+; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v11
+; GFX9-NEXT: v_lshlrev_b64 v[10:11], 1, v[10:11]
+; GFX9-NEXT: v_or_b32_e32 v8, v8, v14
+; GFX9-NEXT: v_or_b32_e32 v14, v13, v11
+; GFX9-NEXT: v_or_b32_e32 v15, v12, v10
; GFX9-NEXT: .LBB1_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v12, 0
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: v_mul_lo_u32 v11, v13, v6
-; GFX9-NEXT: v_mul_lo_u32 v18, v12, v7
-; GFX9-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v5, v12, v[9:10]
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v12, v6, 0
-; GFX9-NEXT: v_mul_lo_u32 v12, v14, v5
-; GFX9-NEXT: v_mul_lo_u32 v15, v15, v4
-; GFX9-NEXT: v_mov_b32_e32 v9, v16
-; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v4, v13, v[9:10]
-; GFX9-NEXT: v_add3_u32 v7, v7, v18, v11
-; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v14, v4, v[6:7]
-; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, v17, v10
-; GFX9-NEXT: v_addc_co_u32_e64 v11, s[4:5], 0, 0, vcc
-; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[10:11]
-; GFX9-NEXT: v_add3_u32 v7, v15, v7, v12
+; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v4, v15, 0
+; GFX9-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-NEXT: v_mul_lo_u32 v13, v14, v6
+; GFX9-NEXT: v_mul_lo_u32 v18, v15, v7
+; GFX9-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v5, v15, v[11:12]
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v15, v6, 0
+; GFX9-NEXT: v_mul_lo_u32 v15, v8, v5
+; GFX9-NEXT: v_mul_lo_u32 v19, v9, v4
+; GFX9-NEXT: v_mov_b32_e32 v11, v16
+; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v4, v14, v[11:12]
+; GFX9-NEXT: v_add3_u32 v7, v7, v18, v13
+; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v8, v4, v[6:7]
+; GFX9-NEXT: v_add_co_u32_e32 v8, vcc, v17, v12
+; GFX9-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
+; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v14, v[8:9]
+; GFX9-NEXT: v_add3_u32 v7, v19, v7, v15
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v7, vcc
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v8
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v9, vcc
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v10
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v11, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v5, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 5aa2cc713a4ec..6690ee8b812e2 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -327,8 +327,14 @@ define void @test_rewrite_mfma_subreg_extract1(float %arg0, float %arg1, ptr add
; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT: s_nop 15
+; CHECK-NEXT: s_nop 1
+; CHECK-NEXT: v_accvgpr_mov_b32 a0, a4
+; CHECK-NEXT: v_accvgpr_mov_b32 a1, a5
+; CHECK-NEXT: v_accvgpr_mov_b32 a2, a6
+; CHECK-NEXT: v_accvgpr_mov_b32 a3, a7
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; use a[4:7]
+; CHECK-NEXT: ; use a[0:3]
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_setpc_b64 s[30:31]
bb:
@@ -552,91 +558,121 @@ define void @test_rewrite_mfma_subreg_insert2(double %arg0, double %arg1, ptr ad
define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) #0 {
; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_accvgpr_write_b32 a34, 2.0
-; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def a[0:31]
; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT: v_accvgpr_read_b32 v63, a31
+; CHECK-NEXT: v_accvgpr_read_b32 v62, a30
+; CHECK-NEXT: v_accvgpr_read_b32 v61, a29
+; CHECK-NEXT: v_accvgpr_read_b32 v60, a28
+; CHECK-NEXT: v_accvgpr_read_b32 v59, a27
+; CHECK-NEXT: v_accvgpr_read_b32 v58, a26
+; CHECK-NEXT: v_accvgpr_read_b32 v57, a25
+; CHECK-NEXT: v_accvgpr_read_b32 v56, a24
+; CHECK-NEXT: v_accvgpr_read_b32 v55, a23
+; CHECK-NEXT: v_accvgpr_read_b32 v54, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v53, a21
+; CHECK-NEXT: v_accvgpr_read_b32 v52, a20
+; CHECK-NEXT: v_accvgpr_read_b32 v51, a19
+; CHECK-NEXT: v_accvgpr_read_b32 v50, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v49, a17
+; CHECK-NEXT: v_accvgpr_read_b32 v48, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v47, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v46, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v45, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v44, a12
+; CHECK-NEXT: v_accvgpr_read_b32 v43, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v42, a10
+; CHECK-NEXT: v_accvgpr_read_b32 v41, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v40, a8
+; CHECK-NEXT: v_accvgpr_read_b32 v39, a7
+; CHECK-NEXT: v_accvgpr_read_b32 v38, a6
+; CHECK-NEXT: v_accvgpr_read_b32 v37, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a4
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a0
+; CHECK-NEXT: v_accvgpr_write_b32 a1, 2.0
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT: v_accvgpr_write_b32 a0, 4.0
; CHECK-NEXT: v_accvgpr_write_b32 a32, v0
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
-; CHECK-NEXT: v_accvgpr_read_b32 v2, a2
-; CHECK-NEXT: v_accvgpr_read_b32 v3, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v4, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a5
-; CHECK-NEXT: v_accvgpr_read_b32 v6, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v7, a7
-; CHECK-NEXT: v_accvgpr_read_b32 v8, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a9
-; CHECK-NEXT: v_accvgpr_read_b32 v10, a10
-; CHECK-NEXT: v_accvgpr_read_b32 v11, a11
-; CHECK-NEXT: v_accvgpr_read_b32 v12, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v13, a13
-; CHECK-NEXT: v_accvgpr_read_b32 v14, a14
-; CHECK-NEXT: v_accvgpr_read_b32 v15, a15
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v17, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v18, a18
-; CHECK-NEXT: v_accvgpr_read_b32 v19, a19
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a21
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a23
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a24
-; CHECK-NEXT: v_accvgpr_read_b32 v25, a25
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a26
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a27
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a29
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
-; CHECK-NEXT: v_accvgpr_write_b32 a33, 4.0
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[32:63], a34, a33, v[0:31]
-; CHECK-NEXT: v_mov_b32_e32 v1, 0x41000000
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], a1, a0, v[32:63]
; CHECK-NEXT: s_nop 15
-; CHECK-NEXT: v_mov_b64_e32 v[2:3], v[32:33]
-; CHECK-NEXT: v_mov_b64_e32 v[4:5], v[34:35]
-; CHECK-NEXT: v_mov_b64_e32 v[6:7], v[36:37]
-; CHECK-NEXT: v_mov_b64_e32 v[8:9], v[38:39]
-; CHECK-NEXT: v_mov_b64_e32 v[10:11], v[40:41]
-; CHECK-NEXT: v_mov_b64_e32 v[12:13], v[42:43]
-; CHECK-NEXT: v_mov_b64_e32 v[14:15], v[44:45]
-; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[46:47]
-; CHECK-NEXT: v_mov_b64_e32 v[18:19], v[48:49]
-; CHECK-NEXT: v_mov_b64_e32 v[20:21], v[50:51]
-; CHECK-NEXT: v_mov_b64_e32 v[22:23], v[52:53]
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[54:55]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[56:57]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[58:59]
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], v[60:61]
-; CHECK-NEXT: v_mov_b64_e32 v[32:33], v[62:63]
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, v[30:33], s[0:1] offset:112
-; CHECK-NEXT: global_store_dwordx4 v0, v[26:29], s[0:1] offset:96
-; CHECK-NEXT: global_store_dwordx4 v0, v[22:25], s[0:1] offset:80
-; CHECK-NEXT: global_store_dwordx4 v0, v[18:21], s[0:1] offset:64
-; CHECK-NEXT: global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
-; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
-; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
-; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
; CHECK-NEXT: s_nop 1
+; CHECK-NEXT: v_accvgpr_write_b32 a0, v0
+; CHECK-NEXT: v_accvgpr_write_b32 a1, v1
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v2
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v4
+; CHECK-NEXT: v_accvgpr_write_b32 a5, v5
+; CHECK-NEXT: v_accvgpr_write_b32 a6, v6
+; CHECK-NEXT: v_accvgpr_write_b32 a7, v7
+; CHECK-NEXT: v_accvgpr_write_b32 a8, v8
+; CHECK-NEXT: v_accvgpr_write_b32 a9, v9
+; CHECK-NEXT: v_accvgpr_write_b32 a10, v10
+; CHECK-NEXT: v_accvgpr_write_b32 a11, v11
+; CHECK-NEXT: v_accvgpr_write_b32 a12, v12
+; CHECK-NEXT: v_accvgpr_write_b32 a13, v13
+; CHECK-NEXT: v_accvgpr_write_b32 a14, v14
+; CHECK-NEXT: v_accvgpr_write_b32 a15, v15
+; CHECK-NEXT: v_accvgpr_write_b32 a16, v16
+; CHECK-NEXT: v_accvgpr_write_b32 a17, v17
+; CHECK-NEXT: v_accvgpr_write_b32 a18, v18
+; CHECK-NEXT: v_accvgpr_write_b32 a19, v19
+; CHECK-NEXT: v_accvgpr_write_b32 a20, v20
+; CHECK-NEXT: v_accvgpr_write_b32 a21, v21
+; CHECK-NEXT: v_accvgpr_write_b32 a22, v22
+; CHECK-NEXT: v_accvgpr_write_b32 a23, v23
+; CHECK-NEXT: v_accvgpr_write_b32 a24, v24
+; CHECK-NEXT: v_accvgpr_write_b32 a25, v25
+; CHECK-NEXT: v_accvgpr_write_b32 a26, v26
+; CHECK-NEXT: v_accvgpr_write_b32 a27, v27
+; CHECK-NEXT: v_accvgpr_write_b32 a28, v28
+; CHECK-NEXT: v_accvgpr_write_b32 a29, v29
+; CHECK-NEXT: v_accvgpr_write_b32 a30, v30
+; CHECK-NEXT: v_accvgpr_write_b32 a31, v31
+; CHECK-NEXT: v_mov_b32_e32 v1, 0x41000000
+; CHECK-NEXT: v_mov_b64_e32 v[4:5], v[32:33]
+; CHECK-NEXT: v_mov_b64_e32 v[6:7], v[34:35]
+; CHECK-NEXT: v_mov_b64_e32 v[8:9], v[36:37]
+; CHECK-NEXT: v_mov_b64_e32 v[10:11], v[38:39]
+; CHECK-NEXT: v_mov_b64_e32 v[12:13], v[40:41]
+; CHECK-NEXT: v_mov_b64_e32 v[14:15], v[42:43]
+; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[44:45]
+; CHECK-NEXT: v_mov_b64_e32 v[18:19], v[46:47]
+; CHECK-NEXT: v_mov_b64_e32 v[20:21], v[48:49]
+; CHECK-NEXT: v_mov_b64_e32 v[22:23], v[50:51]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[52:53]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[54:55]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[56:57]
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], v[58:59]
+; CHECK-NEXT: v_mov_b64_e32 v[32:33], v[60:61]
+; CHECK-NEXT: v_mov_b64_e32 v[34:35], v[62:63]
; CHECK-NEXT: v_mov_b32_e32 v2, 0x41800000
-; CHECK-NEXT: s_nop 1
-; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 a[0:31], v1, v2, a[0:31]
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, a[28:31], s[0:1] offset:112
+; CHECK-NEXT: global_store_dwordx4 v0, a[24:27], s[0:1] offset:96
+; CHECK-NEXT: global_store_dwordx4 v0, a[20:23], s[0:1] offset:80
+; CHECK-NEXT: global_store_dwordx4 v0, a[16:19], s[0:1] offset:64
+; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
+; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
+; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
+; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[4:35], v1, v2, v[4:35]
; CHECK-NEXT: s_nop 15
; CHECK-NEXT: s_nop 1
-; CHECK-NEXT: global_store_dwordx4 v0, a[24:27], s[2:3] offset:96
-; CHECK-NEXT: global_store_dwordx4 v0, a[28:31], s[2:3] offset:112
-; CHECK-NEXT: global_store_dwordx4 v0, a[16:19], s[2:3] offset:64
-; CHECK-NEXT: global_store_dwordx4 v0, a[20:23], s[2:3] offset:80
-; CHECK-NEXT: global_store_dwordx4 v0, a[8:11], s[2:3] offset:32
-; CHECK-NEXT: global_store_dwordx4 v0, a[12:15], s[2:3] offset:48
-; CHECK-NEXT: global_store_dwordx4 v0, a[0:3], s[2:3]
-; CHECK-NEXT: global_store_dwordx4 v0, a[4:7], s[2:3] offset:16
+; CHECK-NEXT: global_store_dwordx4 v0, v[28:31], s[2:3] offset:96
+; CHECK-NEXT: global_store_dwordx4 v0, v[32:35], s[2:3] offset:112
+; CHECK-NEXT: global_store_dwordx4 v0, v[20:23], s[2:3] offset:64
+; CHECK-NEXT: global_store_dwordx4 v0, v[24:27], s[2:3] offset:80
+; CHECK-NEXT: global_store_dwordx4 v0, v[12:15], s[2:3] offset:32
+; CHECK-NEXT: global_store_dwordx4 v0, v[16:19], s[2:3] offset:48
+; CHECK-NEXT: global_store_dwordx4 v0, v[4:7], s[2:3]
+; CHECK-NEXT: global_store_dwordx4 v0, v[8:11], s[2:3] offset:16
; CHECK-NEXT: s_endpgm
%src2 = call <32 x float> asm sideeffect "; def $0", "=a"()
%mai0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %src2, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index 36f073054031a..97f12f67b61e0 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -928,28 +928,28 @@ entry:
define amdgpu_kernel void @select_v2f16_imm_a(
; SI-LABEL: select_v2f16_imm_a:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s11, 0xf000
-; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_mov_b32 s14, s10
-; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s14, s2
+; SI-NEXT: s_mov_b32 s15, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s12, s2
-; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
-; SI-NEXT: s_mov_b32 s16, s4
-; SI-NEXT: s_mov_b32 s17, s5
-; SI-NEXT: s_mov_b32 s18, s10
-; SI-NEXT: s_mov_b32 s19, s11
-; SI-NEXT: s_mov_b32 s4, s6
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: s_mov_b32 s6, s10
-; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_mov_b32 s16, s8
+; SI-NEXT: s_mov_b32 s17, s9
+; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: s_mov_b32 s19, s3
+; SI-NEXT: s_mov_b32 s8, s10
+; SI-NEXT: s_mov_b32 s9, s11
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
-; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
-; SI-NEXT: s_mov_b32 s2, 0x3f200000
-; SI-NEXT: s_mov_b32 s8, s0
-; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s6, 0x3f200000
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cvt_f32_f16_e32 v3, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -957,12 +957,12 @@ define amdgpu_kernel void @select_v2f16_imm_a(
; SI-NEXT: v_cmp_lt_f32_e32 vcc, 0.5, v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
-; SI-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_or_b32_e32 v0, v3, v0
-; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: select_v2f16_imm_a:
@@ -1095,28 +1095,28 @@ entry:
define amdgpu_kernel void @select_v2f16_imm_b(
; SI-LABEL: select_v2f16_imm_b:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s11, 0xf000
-; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_mov_b32 s14, s10
-; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s14, s2
+; SI-NEXT: s_mov_b32 s15, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s12, s2
-; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: s_mov_b32 s12, s6
+; SI-NEXT: s_mov_b32 s13, s7
; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0
-; SI-NEXT: s_mov_b32 s16, s4
-; SI-NEXT: s_mov_b32 s17, s5
-; SI-NEXT: s_mov_b32 s18, s10
-; SI-NEXT: s_mov_b32 s19, s11
-; SI-NEXT: s_mov_b32 s4, s6
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: s_mov_b32 s6, s10
-; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_mov_b32 s16, s8
+; SI-NEXT: s_mov_b32 s17, s9
+; SI-NEXT: s_mov_b32 s18, s2
+; SI-NEXT: s_mov_b32 s19, s3
+; SI-NEXT: s_mov_b32 s8, s10
+; SI-NEXT: s_mov_b32 s9, s11
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0
-; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0
-; SI-NEXT: s_mov_b32 s2, 0x3f200000
-; SI-NEXT: s_mov_b32 s8, s0
-; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: buffer_load_dword v2, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s6, 0x3f200000
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cvt_f32_f16_e32 v3, v0
; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
@@ -1124,12 +1124,12 @@ define amdgpu_kernel void @select_v2f16_imm_b(
; SI-NEXT: v_cmp_gt_f32_e32 vcc, 0.5, v3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
-; SI-NEXT: v_cmp_gt_f32_e32 vcc, s2, v0
+; SI-NEXT: v_cmp_gt_f32_e32 vcc, s6, v0
; SI-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_or_b32_e32 v0, v3, v0
-; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: select_v2f16_imm_b:
@@ -1479,28 +1479,28 @@ define amdgpu_kernel void @select_v2f16_imm_d(
;
; VI-LABEL: select_v2f16_imm_d:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s11, 0xf000
-; VI-NEXT: s_mov_b32 s10, -1
-; VI-NEXT: s_mov_b32 s18, s10
-; VI-NEXT: s_mov_b32 s19, s11
+; VI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s18, s2
+; VI-NEXT: s_mov_b32 s19, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s16, s4
-; VI-NEXT: s_mov_b32 s17, s5
-; VI-NEXT: s_mov_b32 s14, s10
-; VI-NEXT: s_mov_b32 s12, s2
-; VI-NEXT: s_mov_b32 s13, s3
-; VI-NEXT: s_mov_b32 s15, s11
-; VI-NEXT: s_mov_b32 s4, s6
-; VI-NEXT: s_mov_b32 s5, s7
-; VI-NEXT: s_mov_b32 s6, s10
+; VI-NEXT: s_mov_b32 s16, s8
+; VI-NEXT: s_mov_b32 s17, s9
+; VI-NEXT: s_mov_b32 s14, s2
+; VI-NEXT: s_mov_b32 s12, s6
+; VI-NEXT: s_mov_b32 s13, s7
+; VI-NEXT: s_mov_b32 s15, s3
+; VI-NEXT: s_mov_b32 s8, s10
+; VI-NEXT: s_mov_b32 s9, s11
+; VI-NEXT: s_mov_b32 s10, s2
; VI-NEXT: buffer_load_dword v0, off, s[16:19], 0
; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0
-; VI-NEXT: s_mov_b32 s7, s11
-; VI-NEXT: buffer_load_dword v2, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s11, s3
+; VI-NEXT: buffer_load_dword v2, off, s[8:11], 0
; VI-NEXT: v_mov_b32_e32 v3, 0x3800
-; VI-NEXT: s_mov_b32 s8, s0
-; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: s_mov_b32 s0, s4
+; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; VI-NEXT: s_waitcnt vmcnt(1)
@@ -1512,7 +1512,7 @@ define amdgpu_kernel void @select_v2f16_imm_d(
; VI-NEXT: v_mov_b32_e32 v1, 0x3900
; VI-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: select_v2f16_imm_d:
diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll
index acf999e586a68..cdaba4c4f64b1 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -903,28 +903,28 @@ define amdgpu_kernel void @shl_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in
define amdgpu_kernel void @shl_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; SI-LABEL: shl_v4i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s10, s6
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s2
-; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: s_mov_b32 s8, s6
+; SI-NEXT: s_mov_b32 s9, s7
; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
; SI-NEXT: buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
; SI-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
; SI-NEXT: buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshl_b64 v[5:6], v[5:6], v2
; SI-NEXT: v_lshl_b64 v[3:4], v[3:4], v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshl_b64 v[9:10], v[9:10], v13
; SI-NEXT: v_lshl_b64 v[7:8], v[7:8], v11
-; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
+; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: shl_v4i64:
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
index 1c2215d39dc02..acb2e215ddfbd 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
@@ -1840,12 +1840,12 @@ define void @v_shuffle_v2i64_v8i64__9_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v6, v0
+; GFX900-NEXT: v_mov_b32_e32 v7, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -1896,12 +1896,12 @@ define void @v_shuffle_v2i64_v8i64__10_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v8, v0
+; GFX900-NEXT: v_mov_b32_e32 v9, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -1952,12 +1952,12 @@ define void @v_shuffle_v2i64_v8i64__11_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v10, v0
+; GFX900-NEXT: v_mov_b32_e32 v11, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2008,12 +2008,12 @@ define void @v_shuffle_v2i64_v8i64__12_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v12, v0
+; GFX900-NEXT: v_mov_b32_e32 v13, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2064,12 +2064,12 @@ define void @v_shuffle_v2i64_v8i64__13_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v14, v0
+; GFX900-NEXT: v_mov_b32_e32 v15, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2120,12 +2120,12 @@ define void @v_shuffle_v2i64_v8i64__14_0(ptr addrspace(1) inreg %ptr) {
; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[16:31]
+; GFX900-NEXT: ; def v[2:17]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v0
+; GFX900-NEXT: v_mov_b32_e32 v17, v1
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2606,15 +2606,15 @@ define void @v_shuffle_v2i64_v8i64__9_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v8, v2
+; GFX900-NEXT: v_mov_b32_e32 v9, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2662,15 +2662,15 @@ define void @v_shuffle_v2i64_v8i64__10_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v10, v2
+; GFX900-NEXT: v_mov_b32_e32 v11, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2718,15 +2718,15 @@ define void @v_shuffle_v2i64_v8i64__11_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v12, v2
+; GFX900-NEXT: v_mov_b32_e32 v13, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2774,15 +2774,15 @@ define void @v_shuffle_v2i64_v8i64__12_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v14, v2
+; GFX900-NEXT: v_mov_b32_e32 v15, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2830,15 +2830,15 @@ define void @v_shuffle_v2i64_v8i64__13_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v2
+; GFX900-NEXT: v_mov_b32_e32 v17, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -2886,15 +2886,15 @@ define void @v_shuffle_v2i64_v8i64__14_1(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[14:29]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[4:19]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v2
+; GFX900-NEXT: v_mov_b32_e32 v19, v3
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3375,15 +3375,15 @@ define void @v_shuffle_v2i64_v8i64__9_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v10, v4
+; GFX900-NEXT: v_mov_b32_e32 v11, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3431,15 +3431,15 @@ define void @v_shuffle_v2i64_v8i64__10_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v12, v4
+; GFX900-NEXT: v_mov_b32_e32 v13, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3487,15 +3487,15 @@ define void @v_shuffle_v2i64_v8i64__11_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v14, v4
+; GFX900-NEXT: v_mov_b32_e32 v15, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3543,15 +3543,15 @@ define void @v_shuffle_v2i64_v8i64__12_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v4
+; GFX900-NEXT: v_mov_b32_e32 v17, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3599,15 +3599,15 @@ define void @v_shuffle_v2i64_v8i64__13_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v4
+; GFX900-NEXT: v_mov_b32_e32 v19, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -3655,15 +3655,15 @@ define void @v_shuffle_v2i64_v8i64__14_2(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[12:27]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[6:21]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v4
+; GFX900-NEXT: v_mov_b32_e32 v21, v5
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4144,15 +4144,15 @@ define void @v_shuffle_v2i64_v8i64__9_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v12, v6
+; GFX900-NEXT: v_mov_b32_e32 v13, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4200,15 +4200,15 @@ define void @v_shuffle_v2i64_v8i64__10_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v14, v6
+; GFX900-NEXT: v_mov_b32_e32 v15, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4256,15 +4256,15 @@ define void @v_shuffle_v2i64_v8i64__11_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v6
+; GFX900-NEXT: v_mov_b32_e32 v17, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4312,15 +4312,15 @@ define void @v_shuffle_v2i64_v8i64__12_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v6
+; GFX900-NEXT: v_mov_b32_e32 v19, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4368,15 +4368,15 @@ define void @v_shuffle_v2i64_v8i64__13_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v6
+; GFX900-NEXT: v_mov_b32_e32 v21, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4424,15 +4424,15 @@ define void @v_shuffle_v2i64_v8i64__14_3(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[10:25]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[8:23]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v22, v6
+; GFX900-NEXT: v_mov_b32_e32 v23, v7
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[20:23], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4913,15 +4913,15 @@ define void @v_shuffle_v2i64_v8i64__9_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v14, v8
+; GFX900-NEXT: v_mov_b32_e32 v15, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -4969,15 +4969,15 @@ define void @v_shuffle_v2i64_v8i64__10_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v8
+; GFX900-NEXT: v_mov_b32_e32 v17, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5025,15 +5025,15 @@ define void @v_shuffle_v2i64_v8i64__11_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v8
+; GFX900-NEXT: v_mov_b32_e32 v19, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5081,15 +5081,15 @@ define void @v_shuffle_v2i64_v8i64__12_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v8
+; GFX900-NEXT: v_mov_b32_e32 v21, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5137,15 +5137,15 @@ define void @v_shuffle_v2i64_v8i64__13_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v22, v8
+; GFX900-NEXT: v_mov_b32_e32 v23, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[20:23], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5193,15 +5193,15 @@ define void @v_shuffle_v2i64_v8i64__14_4(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[8:23]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[10:25]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v24, v8
+; GFX900-NEXT: v_mov_b32_e32 v25, v9
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5682,15 +5682,15 @@ define void @v_shuffle_v2i64_v8i64__9_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v16, v10
+; GFX900-NEXT: v_mov_b32_e32 v17, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5738,15 +5738,15 @@ define void @v_shuffle_v2i64_v8i64__10_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v10
+; GFX900-NEXT: v_mov_b32_e32 v19, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5794,15 +5794,15 @@ define void @v_shuffle_v2i64_v8i64__11_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v10
+; GFX900-NEXT: v_mov_b32_e32 v21, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5850,15 +5850,15 @@ define void @v_shuffle_v2i64_v8i64__12_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v22, v10
+; GFX900-NEXT: v_mov_b32_e32 v23, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[20:23], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5906,15 +5906,15 @@ define void @v_shuffle_v2i64_v8i64__13_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v24, v10
+; GFX900-NEXT: v_mov_b32_e32 v25, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -5962,15 +5962,15 @@ define void @v_shuffle_v2i64_v8i64__14_5(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[6:21]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[12:27]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v26, v10
+; GFX900-NEXT: v_mov_b32_e32 v27, v11
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[24:27], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6451,15 +6451,15 @@ define void @v_shuffle_v2i64_v8i64__9_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v18, v12
+; GFX900-NEXT: v_mov_b32_e32 v19, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[16:19], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6507,15 +6507,15 @@ define void @v_shuffle_v2i64_v8i64__10_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v12
+; GFX900-NEXT: v_mov_b32_e32 v21, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6563,15 +6563,15 @@ define void @v_shuffle_v2i64_v8i64__11_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v22, v12
+; GFX900-NEXT: v_mov_b32_e32 v23, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[20:23], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6619,15 +6619,15 @@ define void @v_shuffle_v2i64_v8i64__12_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v24, v12
+; GFX900-NEXT: v_mov_b32_e32 v25, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6675,15 +6675,15 @@ define void @v_shuffle_v2i64_v8i64__13_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v26, v12
+; GFX900-NEXT: v_mov_b32_e32 v27, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[24:27], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -6731,15 +6731,15 @@ define void @v_shuffle_v2i64_v8i64__14_6(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[4:19]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[14:29]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v28, v12
+; GFX900-NEXT: v_mov_b32_e32 v29, v13
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[26:29], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7220,15 +7220,15 @@ define void @v_shuffle_v2i64_v8i64__9_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v4, v16
-; GFX900-NEXT: v_mov_b32_e32 v5, v17
+; GFX900-NEXT: v_mov_b32_e32 v20, v14
+; GFX900-NEXT: v_mov_b32_e32 v21, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7276,15 +7276,15 @@ define void @v_shuffle_v2i64_v8i64__10_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v6, v16
-; GFX900-NEXT: v_mov_b32_e32 v7, v17
+; GFX900-NEXT: v_mov_b32_e32 v22, v14
+; GFX900-NEXT: v_mov_b32_e32 v23, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[4:7], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[20:23], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7332,15 +7332,15 @@ define void @v_shuffle_v2i64_v8i64__11_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v8, v16
-; GFX900-NEXT: v_mov_b32_e32 v9, v17
+; GFX900-NEXT: v_mov_b32_e32 v24, v14
+; GFX900-NEXT: v_mov_b32_e32 v25, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7388,15 +7388,15 @@ define void @v_shuffle_v2i64_v8i64__12_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v10, v16
-; GFX900-NEXT: v_mov_b32_e32 v11, v17
+; GFX900-NEXT: v_mov_b32_e32 v26, v14
+; GFX900-NEXT: v_mov_b32_e32 v27, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[24:27], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7444,15 +7444,15 @@ define void @v_shuffle_v2i64_v8i64__13_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v12, v16
-; GFX900-NEXT: v_mov_b32_e32 v13, v17
+; GFX900-NEXT: v_mov_b32_e32 v28, v14
+; GFX900-NEXT: v_mov_b32_e32 v29, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[26:29], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
@@ -7500,15 +7500,15 @@ define void @v_shuffle_v2i64_v8i64__14_7(ptr addrspace(1) inreg %ptr) {
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[2:17]
+; GFX900-NEXT: ; def v[0:15]
; GFX900-NEXT: ;;#ASMEND
; GFX900-NEXT: ;;#ASMSTART
-; GFX900-NEXT: ; def v[0:15]
+; GFX900-NEXT: ; def v[16:31]
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v14, v16
-; GFX900-NEXT: v_mov_b32_e32 v15, v17
+; GFX900-NEXT: v_mov_b32_e32 v30, v14
+; GFX900-NEXT: v_mov_b32_e32 v31, v15
; GFX900-NEXT: v_mov_b32_e32 v0, 0
-; GFX900-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
+; GFX900-NEXT: global_store_dwordx4 v0, v[28:31], s[16:17]
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index b5474b8974b29..ba996fe48f31d 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -4672,14 +4672,15 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0
; GFX9-FLATSCR-NEXT: v_lshlrev_b32_e32 v5, 13, v0
-; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s8, s13
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x80
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v2, vcc, s2, v5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s3
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v0, vcc
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x80, v2
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
+; GFX9-FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s8, s13
; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
; GFX9-FLATSCR-NEXT: s_mov_b32 s4, 4
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4709,11 +4710,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x74
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x100
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x84
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4743,11 +4745,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x180, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0xf4
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x180
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x104
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4777,11 +4780,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x200, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x174
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x200
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x184
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4811,11 +4815,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x280, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x1f4
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x280
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x204
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4845,11 +4850,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x300, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x274
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x300
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x284
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4879,11 +4885,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x380, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x2f4
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x380
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x304
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -4913,11 +4920,12 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:4080
-; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 0x400, v2
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x374
-; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s4 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x400
+; GFX9-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
+; GFX9-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:3968
; GFX9-FLATSCR-NEXT: s_movk_i32 s4, 0x384
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
@@ -9771,7 +9779,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
; GFX6-NEXT: v_lshlrev_b32_e32 v5, 8, v0
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:240
-; GFX6-NEXT: s_mov_b32 s2, 0x86a00
+; GFX6-NEXT: s_mov_b32 s2, 0x83a00
; GFX6-NEXT: s_mov_b64 s[8:9], exec
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
@@ -9780,7 +9788,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:224
-; GFX6-NEXT: s_mov_b32 s2, 0x86600
+; GFX6-NEXT: s_mov_b32 s2, 0x83e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9788,7 +9796,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:208
-; GFX6-NEXT: s_mov_b32 s2, 0x86200
+; GFX6-NEXT: s_mov_b32 s2, 0x84200
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9796,7 +9804,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:192
-; GFX6-NEXT: s_mov_b32 s2, 0x85e00
+; GFX6-NEXT: s_mov_b32 s2, 0x84600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9804,7 +9812,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:176
-; GFX6-NEXT: s_mov_b32 s2, 0x85a00
+; GFX6-NEXT: s_mov_b32 s2, 0x84a00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9812,7 +9820,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:160
-; GFX6-NEXT: s_mov_b32 s2, 0x85600
+; GFX6-NEXT: s_mov_b32 s2, 0x84e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9828,7 +9836,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:128
-; GFX6-NEXT: s_mov_b32 s2, 0x84e00
+; GFX6-NEXT: s_mov_b32 s2, 0x85600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9836,7 +9844,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:112
-; GFX6-NEXT: s_mov_b32 s2, 0x84a00
+; GFX6-NEXT: s_mov_b32 s2, 0x85a00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9844,7 +9852,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:96
-; GFX6-NEXT: s_mov_b32 s2, 0x84600
+; GFX6-NEXT: s_mov_b32 s2, 0x85e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9852,7 +9860,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:80
-; GFX6-NEXT: s_mov_b32 s2, 0x84200
+; GFX6-NEXT: s_mov_b32 s2, 0x86200
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9860,7 +9868,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64 offset:64
-; GFX6-NEXT: s_mov_b32 s2, 0x83a00
+; GFX6-NEXT: s_mov_b32 s2, 0x86600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
@@ -9898,7 +9906,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[8:9]
; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:48
-; GFX6-NEXT: s_mov_b32 s0, 0x83e00
+; GFX6-NEXT: s_mov_b32 s0, 0x86a00
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 13, v0
; GFX6-NEXT: v_add_i32_e32 v4, vcc, 16, v4
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -10179,13 +10187,13 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_mov_b64 exec, s[4:5]
-; GFX6-NEXT: s_mov_b32 s0, 0x86a00
+; GFX6-NEXT: s_mov_b32 s0, 0x83a00
; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
; GFX6-NEXT: s_mov_b64 s[38:39], s[2:3]
-; GFX6-NEXT: s_mov_b32 s0, 0x86600
+; GFX6-NEXT: s_mov_b32 s0, 0x83e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10193,7 +10201,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x86200
+; GFX6-NEXT: s_mov_b32 s0, 0x84200
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:224
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10201,7 +10209,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85e00
+; GFX6-NEXT: s_mov_b32 s0, 0x84600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:208
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10209,7 +10217,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85a00
+; GFX6-NEXT: s_mov_b32 s0, 0x84a00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:192
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10217,7 +10225,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x85600
+; GFX6-NEXT: s_mov_b32 s0, 0x84e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:176
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10233,7 +10241,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84e00
+; GFX6-NEXT: s_mov_b32 s0, 0x85600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:144
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10241,7 +10249,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84a00
+; GFX6-NEXT: s_mov_b32 s0, 0x85a00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:128
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10249,7 +10257,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84600
+; GFX6-NEXT: s_mov_b32 s0, 0x85e00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:112
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10257,7 +10265,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x84200
+; GFX6-NEXT: s_mov_b32 s0, 0x86200
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:96
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10265,7 +10273,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83a00
+; GFX6-NEXT: s_mov_b32 s0, 0x86600
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:80
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10273,7 +10281,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b32 s0, 0x83e00
+; GFX6-NEXT: s_mov_b32 s0, 0x86a00
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:64
; GFX6-NEXT: s_waitcnt expcnt(0)
@@ -10312,60 +10320,56 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:240
; GFX9-FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:192
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v4, 16
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:144
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX9-FLATSCR-NEXT: s_movk_i32 s34, 0x20c0
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:208
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:128
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(3)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:224
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:208
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:176
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:192
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:176
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:160
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:128
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:112
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:96
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:32
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:16
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:144
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(2)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:80
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:64
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:160
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:48
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[19:22], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:112
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:96
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[7:10], v5, s[38:39]
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[7:10], v5, s[38:39] offset:80
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:64
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v6, 1
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: v_lshl_add_u32 v4, v7, 13, v4
-; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:48
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:32
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[15:18], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:16
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39]
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: v_lshl_add_u32 v4, v0, 13, v4
+; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: scratch_store_dword v4, v6, off
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ; def s[0:7]
@@ -10388,6 +10392,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ; def s33
; GFX9-FLATSCR-NEXT: ;;#ASMEND
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s34 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[34:35], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB1_2
; GFX9-FLATSCR-NEXT: ; %bb.1: ; %bb0
@@ -10395,26 +10400,27 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: ; use s[0:7],s[8:15],s[16:23],s[24:31],s[40:43],s[38:39]
; GFX9-FLATSCR-NEXT: ;;#ASMEND
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20d0
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[15:18], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[19:22], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, v19
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[11:14], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20f0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, v11
-; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[7:10], s0 ; 16-byte Folded Spill
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v12
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v13
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v14
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, v20
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, v21
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v3, v22
+; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[15:18], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: s_nop 0
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ;;#ASMEND
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20e0
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20d0
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[15:18], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v14, v3
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v13, v2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v12, v1
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v11, v0
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[7:10], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v22, v3
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v21, v2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v20, v1
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v19, v0
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
; GFX9-FLATSCR-NEXT: ;;#ASMEND
; GFX9-FLATSCR-NEXT: ;;#ASMSTART
@@ -10429,53 +10435,53 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: ;;#ASMEND
; GFX9-FLATSCR-NEXT: .LBB1_2: ; %ret
; GFX9-FLATSCR-NEXT: s_or_b64 exec, exec, s[34:35]
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:112
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:112
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[19:22], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20a0
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:96
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2090
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:96
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37] offset:80
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:80
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:64
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:48
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20c0
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:32
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:16
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:64
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37]
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2060
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:240
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:224
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:48
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:32
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[11:14], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:16
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37]
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:208
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2040
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:240
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:192
+; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2050
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:224
-; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[6:9], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
+; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:176
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[6:9], s[36:37] offset:208
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:192
-; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:176
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(3)
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2080
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:160
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
-; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
+; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37] offset:144
; GFX9-FLATSCR-NEXT: scratch_load_dwordx4 v[0:3], off, s0 ; 16-byte Folded Reload
@@ -10497,21 +10503,21 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v5, 8, v0
; GFX10-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FLATSCR-NEXT: s_clause 0xf
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[35:38], v5, s[38:39] offset:240
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[31:34], v5, s[38:39] offset:224
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[27:30], v5, s[38:39] offset:208
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[23:26], v5, s[38:39] offset:192
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:176
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:160
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:144
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[7:10], v5, s[38:39] offset:128
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[63:66], v5, s[38:39] offset:112
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[59:62], v5, s[38:39] offset:96
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[55:58], v5, s[38:39] offset:80
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[7:10], v5, s[38:39] offset:240
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:224
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:208
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:192
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[23:26], v5, s[38:39] offset:176
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[27:30], v5, s[38:39] offset:160
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[31:34], v5, s[38:39] offset:144
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[35:38], v5, s[38:39] offset:128
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[39:42], v5, s[38:39] offset:112
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[43:46], v5, s[38:39] offset:96
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[47:50], v5, s[38:39] offset:80
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[51:54], v5, s[38:39] offset:64
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[47:50], v5, s[38:39] offset:48
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[43:46], v5, s[38:39] offset:32
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[39:42], v5, s[38:39] offset:16
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[55:58], v5, s[38:39] offset:48
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[59:62], v5, s[38:39] offset:32
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[63:66], v5, s[38:39] offset:16
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39]
; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_lshl_add_u32 v4, v0, 13, 16
@@ -10674,23 +10680,23 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: ;;#ASMEND
; GFX10-FLATSCR-NEXT: .LBB1_2: ; %ret
; GFX10-FLATSCR-NEXT: s_or_b32 exec_lo, exec_lo, s33
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[63:66], s[36:37] offset:112
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[59:62], s[36:37] offset:96
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[55:58], s[36:37] offset:80
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[39:42], s[36:37] offset:112
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[43:46], s[36:37] offset:96
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[47:50], s[36:37] offset:80
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[51:54], s[36:37] offset:64
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[47:50], s[36:37] offset:48
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[43:46], s[36:37] offset:32
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[39:42], s[36:37] offset:16
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[55:58], s[36:37] offset:48
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[59:62], s[36:37] offset:32
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[63:66], s[36:37] offset:16
; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[0:3], s[36:37]
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[35:38], s[36:37] offset:240
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[31:34], s[36:37] offset:224
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[27:30], s[36:37] offset:208
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[23:26], s[36:37] offset:192
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:176
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:160
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:144
-; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37] offset:128
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[7:10], s[36:37] offset:240
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[11:14], s[36:37] offset:224
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[15:18], s[36:37] offset:208
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[19:22], s[36:37] offset:192
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[23:26], s[36:37] offset:176
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[27:30], s[36:37] offset:160
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[31:34], s[36:37] offset:144
+; GFX10-FLATSCR-NEXT: global_store_dwordx4 v5, v[35:38], s[36:37] offset:128
; GFX10-FLATSCR-NEXT: s_endpgm
entry:
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 2b20f9d545899..4bdfc5407dabd 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -85,13 +85,13 @@ define amdgpu_kernel void @max_10_vgprs_spill_v32(ptr addrspace(1) %p) #0 {
; GFX908-DAG: v_accvgpr_read_b32
; GFX900: NumVgprs: 256
-; GFX900: ScratchSize: 132
-; GFX908: NumVgprs: 252
+; GFX900: ScratchSize: 148
+; GFX908: NumVgprs: 254
; GFX908: ScratchSize: 0
; GFX900: VGPRBlocks: 63
-; GFX908: VGPRBlocks: 62
+; GFX908: VGPRBlocks: 63
; GFX900: NumVGPRsForWavesPerEU: 256
-; GFX908: NumVGPRsForWavesPerEU: 252
+; GFX908: NumVGPRsForWavesPerEU: 254
define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%p1 = getelementptr inbounds <32 x float>, ptr addrspace(1) %p, i32 %tid
@@ -136,13 +136,13 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
; GFX908-DAG: v_accvgpr_read_b32
; GFX900: NumVgprs: 256
-; GFX908: NumVgprs: 254
-; GFX900: ScratchSize: 132
+; GFX908: NumVgprs: 252
+; GFX900: ScratchSize: 148
; GFX908: ScratchSize: 0
; GFX900: VGPRBlocks: 63
-; GFX908: VGPRBlocks: 63
+; GFX908: VGPRBlocks: 62
; GFX900: NumVGPRsForWavesPerEU: 256
-; GFX908: NumVGPRsForWavesPerEU: 254
+; GFX908: NumVGPRsForWavesPerEU: 252
define amdgpu_kernel void @max_256_vgprs_spill_9x32_2bb(ptr addrspace(1) %p) #1 {
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%p1 = getelementptr inbounds <32 x float>, ptr addrspace(1) %p, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index 4cf74a9ba96a8..14d0a82378c47 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -594,28 +594,28 @@ define amdgpu_kernel void @ashr_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %i
define amdgpu_kernel void @ashr_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; SI-LABEL: ashr_v4i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s10, s6
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s2
-; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: s_mov_b32 s8, s6
+; SI-NEXT: s_mov_b32 s9, s7
; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
; SI-NEXT: buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
; SI-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
; SI-NEXT: buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_ashr_i64 v[5:6], v[5:6], v2
; SI-NEXT: v_ashr_i64 v[3:4], v[3:4], v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_ashr_i64 v[9:10], v[9:10], v13
; SI-NEXT: v_ashr_i64 v[7:8], v[7:8], v11
-; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
+; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: ashr_v4i64:
diff --git a/llvm/test/CodeGen/AMDGPU/srl.ll b/llvm/test/CodeGen/AMDGPU/srl.ll
index 571c0f04c06ca..6672d4a3b3281 100644
--- a/llvm/test/CodeGen/AMDGPU/srl.ll
+++ b/llvm/test/CodeGen/AMDGPU/srl.ll
@@ -258,28 +258,28 @@ define amdgpu_kernel void @lshr_i64(ptr addrspace(1) %out, ptr addrspace(1) %in)
define amdgpu_kernel void @lshr_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; SI-LABEL: lshr_v4i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s10, s6
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s10, s2
+; SI-NEXT: s_mov_b32 s11, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s2
-; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: s_mov_b32 s8, s6
+; SI-NEXT: s_mov_b32 s9, s7
; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:48
; SI-NEXT: buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
; SI-NEXT: buffer_load_dwordx4 v[7:10], off, s[8:11], 0
; SI-NEXT: buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], v2
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], v13
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], v11
-; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0 offset:16
-; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
+; SI-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:16
+; SI-NEXT: buffer_store_dwordx4 v[7:10], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: lshr_v4i64:
diff --git a/llvm/test/CodeGen/AMDGPU/sub_i1.ll b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
index 3ef1cece538ed..f58367d7063fc 100644
--- a/llvm/test/CodeGen/AMDGPU/sub_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
@@ -111,25 +111,25 @@ define amdgpu_kernel void @sub_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a,
; GFX9-LABEL: sub_i1_cf:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX9-NEXT: ; implicit-def: $sgpr4_sgpr5
-; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX9-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_xor_b64 s[8:9], exec, s[8:9]
; GFX9-NEXT: s_cbranch_execz .LBB2_2
; GFX9-NEXT: ; %bb.1: ; %else
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT: global_load_ubyte v0, v0, s[6:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0
; GFX9-NEXT: .LBB2_2: ; %Flow
-; GFX9-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_andn2_saveexec_b64 s[6:7], s[8:9]
; GFX9-NEXT: s_cbranch_execz .LBB2_4
; GFX9-NEXT: ; %bb.3: ; %if
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_andn2_b64 s[2:3], s[4:5], exec
@@ -139,7 +139,6 @@ define amdgpu_kernel void @sub_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a,
; GFX9-NEXT: s_or_b64 s[4:5], s[2:3], s[4:5]
; GFX9-NEXT: .LBB2_4: ; %endif
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b64 s[2:3], s[4:5], -1
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index dd2acb8de6f41..811f229d6f990 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -2362,18 +2362,19 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_mov_b32 s4, 0
+; SI-NEXT: s_mov_b32 s5, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_load_sbyte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s6, s2
; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_cvt_f32_i32_e32 v2, v0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; SI-NEXT: v_xor_b32_e32 v0, v1, v0
; SI-NEXT: v_rcp_iflag_f32_e32 v4, v2
+; SI-NEXT: v_xor_b32_e32 v0, v1, v0
; SI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
; SI-NEXT: v_or_b32_e32 v0, 1, v0
; SI-NEXT: v_mul_f32_e32 v1, v3, v4
@@ -2391,18 +2392,19 @@ define amdgpu_kernel void @fdiv_test_denormals(ptr addrspace(1) nocapture readon
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b64 s[4:5], 0
-; VI-NEXT: s_mov_b32 s6, s2
+; VI-NEXT: s_mov_b32 s4, 0
+; VI-NEXT: s_mov_b32 s5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_load_sbyte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s6, s2
; VI-NEXT: s_mov_b32 s7, s3
; VI-NEXT: buffer_load_sbyte v1, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_i32_e32 v2, v0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cvt_f32_i32_e32 v3, v1
-; VI-NEXT: v_xor_b32_e32 v0, v1, v0
; VI-NEXT: v_rcp_iflag_f32_e32 v4, v2
+; VI-NEXT: v_xor_b32_e32 v0, v1, v0
; VI-NEXT: v_ashrrev_i32_e32 v0, 30, v0
; VI-NEXT: v_or_b32_e32 v0, 1, v0
; VI-NEXT: v_mul_f32_e32 v1, v3, v4
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.i64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
index bf57f65a570cf..8bdce9c7fce04 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
@@ -597,32 +597,32 @@ define amdgpu_kernel void @v_uint_to_fp_v4i64_to_v4f32(ptr addrspace(1) %out, pt
define amdgpu_kernel void @s_uint_to_fp_v2i64_to_v2f16(ptr addrspace(1) %out, <2 x i64> %in) #0{
; GFX6-LABEL: s_uint_to_fp_v2i64_to_v2f16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_flbit_i32_b32 s8, s3
-; GFX6-NEXT: s_flbit_i32_b32 s9, s1
-; GFX6-NEXT: s_min_u32 s8, s8, 32
-; GFX6-NEXT: s_min_u32 s9, s9, 32
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s8
-; GFX6-NEXT: s_sub_i32 s8, 32, s8
-; GFX6-NEXT: s_lshl_b64 s[0:1], s[0:1], s9
-; GFX6-NEXT: s_sub_i32 s9, 32, s9
-; GFX6-NEXT: s_min_u32 s2, s2, 1
-; GFX6-NEXT: s_min_u32 s0, s0, 1
-; GFX6-NEXT: s_or_b32 s2, s3, s2
-; GFX6-NEXT: s_or_b32 s0, s1, s0
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
-; GFX6-NEXT: v_ldexp_f32_e64 v0, v0, s8
-; GFX6-NEXT: v_ldexp_f32_e64 v1, v1, s9
+; GFX6-NEXT: s_flbit_i32_b32 s4, s11
+; GFX6-NEXT: s_flbit_i32_b32 s5, s9
+; GFX6-NEXT: s_min_u32 s6, s4, 32
+; GFX6-NEXT: s_min_u32 s12, s5, 32
+; GFX6-NEXT: s_lshl_b64 s[4:5], s[10:11], s6
+; GFX6-NEXT: s_sub_i32 s10, 32, s6
+; GFX6-NEXT: s_lshl_b64 s[6:7], s[8:9], s12
+; GFX6-NEXT: s_sub_i32 s8, 32, s12
+; GFX6-NEXT: s_min_u32 s4, s4, 1
+; GFX6-NEXT: s_min_u32 s6, s6, 1
+; GFX6-NEXT: s_or_b32 s4, s5, s4
+; GFX6-NEXT: s_or_b32 s5, s7, s6
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s5
+; GFX6-NEXT: v_ldexp_f32_e64 v0, v0, s10
+; GFX6-NEXT: v_ldexp_f32_e64 v1, v1, s8
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_uint_to_fp_v2i64_to_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/unspill-vgpr-after-rewrite-vgpr-mfma.ll b/llvm/test/CodeGen/AMDGPU/unspill-vgpr-after-rewrite-vgpr-mfma.ll
index a81d9a458e23a..e9f7e3d2e1f10 100644
--- a/llvm/test/CodeGen/AMDGPU/unspill-vgpr-after-rewrite-vgpr-mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/unspill-vgpr-after-rewrite-vgpr-mfma.ll
@@ -12,97 +12,74 @@ define void @eliminate_spill_after_mfma_rewrite(i32 %x, i32 %y, <4 x i32> %arg,
; CHECK-NEXT: v_accvgpr_write_b32 a2, v4
; CHECK-NEXT: v_accvgpr_write_b32 a1, v3
; CHECK-NEXT: v_accvgpr_write_b32 a0, v2
-; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a40, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a47, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a48, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a49, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a50, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a51, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a52, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a53, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a54, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a55, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a63, off, s[0:3], s32 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 ; 4-byte Folded Spill
; CHECK-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[32:63], v[0:31]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: v_accvgpr_write_b32 a63, v31
-; CHECK-NEXT: v_accvgpr_write_b32 a62, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a61, v29
-; CHECK-NEXT: v_accvgpr_write_b32 a60, v28
-; CHECK-NEXT: v_accvgpr_write_b32 a59, v27
-; CHECK-NEXT: v_accvgpr_write_b32 a58, v26
-; CHECK-NEXT: v_accvgpr_write_b32 a57, v25
-; CHECK-NEXT: v_accvgpr_write_b32 a56, v24
-; CHECK-NEXT: v_accvgpr_write_b32 a55, v23
-; CHECK-NEXT: v_accvgpr_write_b32 a54, v22
-; CHECK-NEXT: v_accvgpr_write_b32 a53, v21
-; CHECK-NEXT: v_accvgpr_write_b32 a52, v20
-; CHECK-NEXT: v_accvgpr_write_b32 a51, v19
-; CHECK-NEXT: v_accvgpr_write_b32 a50, v18
-; CHECK-NEXT: v_accvgpr_write_b32 a49, v17
-; CHECK-NEXT: v_accvgpr_write_b32 a48, v16
-; CHECK-NEXT: v_accvgpr_write_b32 a47, v15
-; CHECK-NEXT: v_accvgpr_write_b32 a46, v14
-; CHECK-NEXT: v_accvgpr_write_b32 a45, v13
-; CHECK-NEXT: v_accvgpr_write_b32 a44, v12
-; CHECK-NEXT: v_accvgpr_write_b32 a43, v11
-; CHECK-NEXT: v_accvgpr_write_b32 a42, v10
-; CHECK-NEXT: v_accvgpr_write_b32 a41, v9
-; CHECK-NEXT: v_accvgpr_write_b32 a40, v8
-; CHECK-NEXT: v_accvgpr_write_b32 a39, v7
-; CHECK-NEXT: v_accvgpr_write_b32 a38, v6
-; CHECK-NEXT: v_accvgpr_write_b32 a37, v5
-; CHECK-NEXT: v_accvgpr_write_b32 a36, v4
-; CHECK-NEXT: v_accvgpr_write_b32 a35, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a34, v2
-; CHECK-NEXT: v_accvgpr_write_b32 a33, v1
-; CHECK-NEXT: v_accvgpr_write_b32 a32, v0
+; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
; CHECK-NEXT: v_accvgpr_read_b32 v2, a2
; CHECK-NEXT: v_accvgpr_read_b32 v3, a3
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; def v[6:9]
+; CHECK-NEXT: ; def v[0:3]
; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: v_accvgpr_write_b32 a35, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a34, v2
+; CHECK-NEXT: v_accvgpr_write_b32 a33, v1
+; CHECK-NEXT: v_accvgpr_write_b32 a32, v0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def a[0:31]
@@ -126,72 +103,77 @@ define void @eliminate_spill_after_mfma_rewrite(i32 %x, i32 %y, <4 x i32> %arg,
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v0, v[32:35], s[16:17]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[56:59], s[16:17] offset:96
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[60:63], s[16:17] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[48:51], s[16:17] offset:64
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[52:55], s[16:17] offset:80
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[40:43], s[16:17] offset:32
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[44:47], s[16:17] offset:48
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(4)
+; CHECK-NEXT: global_store_dwordx4 v0, v[26:29], s[16:17] offset:96
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[30:33], s[16:17] offset:112
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17] offset:80
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17] offset:48
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v0, a[32:35], s[16:17]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[36:39], s[16:17] offset:16
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17]
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
%mai = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 %x, i32 %y, <4 x i32> %arg, i32 0, i32 0, i32 0)
@@ -216,101 +198,86 @@ define void @eliminate_spill_after_mfma_rewrite_x2(i32 %x, i32 %y, <4 x i32> %ar
; CHECK-NEXT: v_accvgpr_write_b32 a2, v4
; CHECK-NEXT: v_accvgpr_write_b32 a1, v3
; CHECK-NEXT: v_accvgpr_write_b32 a0, v2
-; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a40, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a47, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a48, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a49, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a50, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a51, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a52, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a53, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a54, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a55, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a56, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a57, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a58, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a59, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a60, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a61, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a62, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_store_dword a63, off, s[0:3], s32 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a34, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a35, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a36, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a37, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a38, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword a39, off, s[0:3], s32 ; 4-byte Folded Spill
; CHECK-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def v[32:63], v[0:31]
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: v_accvgpr_write_b32 a63, v31
-; CHECK-NEXT: v_accvgpr_write_b32 a62, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a61, v29
-; CHECK-NEXT: v_accvgpr_write_b32 a60, v28
-; CHECK-NEXT: v_accvgpr_write_b32 a59, v27
-; CHECK-NEXT: v_accvgpr_write_b32 a58, v26
-; CHECK-NEXT: v_accvgpr_write_b32 a57, v25
-; CHECK-NEXT: v_accvgpr_write_b32 a56, v24
-; CHECK-NEXT: v_accvgpr_write_b32 a55, v23
-; CHECK-NEXT: v_accvgpr_write_b32 a54, v22
-; CHECK-NEXT: v_accvgpr_write_b32 a53, v21
-; CHECK-NEXT: v_accvgpr_write_b32 a52, v20
-; CHECK-NEXT: v_accvgpr_write_b32 a51, v19
-; CHECK-NEXT: v_accvgpr_write_b32 a50, v18
-; CHECK-NEXT: v_accvgpr_write_b32 a49, v17
-; CHECK-NEXT: v_accvgpr_write_b32 a48, v16
-; CHECK-NEXT: v_accvgpr_write_b32 a47, v15
-; CHECK-NEXT: v_accvgpr_write_b32 a46, v14
-; CHECK-NEXT: v_accvgpr_write_b32 a45, v13
-; CHECK-NEXT: v_accvgpr_write_b32 a44, v12
-; CHECK-NEXT: v_accvgpr_write_b32 a43, v11
-; CHECK-NEXT: v_accvgpr_write_b32 a42, v10
-; CHECK-NEXT: v_accvgpr_write_b32 a41, v9
-; CHECK-NEXT: v_accvgpr_write_b32 a40, v8
-; CHECK-NEXT: v_accvgpr_write_b32 a39, v7
-; CHECK-NEXT: v_accvgpr_write_b32 a38, v6
-; CHECK-NEXT: v_accvgpr_write_b32 a37, v5
-; CHECK-NEXT: v_accvgpr_write_b32 a36, v4
-; CHECK-NEXT: v_accvgpr_write_b32 a35, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a34, v2
-; CHECK-NEXT: v_accvgpr_write_b32 a33, v1
-; CHECK-NEXT: v_accvgpr_write_b32 a32, v0
+; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; CHECK-NEXT: v_accvgpr_read_b32 v7, a3
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_accvgpr_read_b32 v6, a2
; CHECK-NEXT: v_accvgpr_read_b32 v5, a1
; CHECK-NEXT: v_accvgpr_read_b32 v4, a0
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; def v[8:11]
+; CHECK-NEXT: ; def v[0:3]
; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: v_accvgpr_write_b32 a35, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a34, v2
+; CHECK-NEXT: v_accvgpr_write_b32 a33, v1
+; CHECK-NEXT: v_accvgpr_write_b32 a32, v0
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ; def v[12:15]
+; CHECK-NEXT: ; def v[0:3]
; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: v_accvgpr_write_b32 a39, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a38, v2
+; CHECK-NEXT: v_accvgpr_write_b32 a37, v1
+; CHECK-NEXT: v_accvgpr_write_b32 a36, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; def a[0:31]
; CHECK-NEXT: ;;#ASMEND
@@ -333,74 +300,83 @@ define void @eliminate_spill_after_mfma_rewrite_x2(i32 %x, i32 %y, <4 x i32> %ar
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v0, v[32:35], s[16:17]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[56:59], s[16:17] offset:96
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[60:63], s[16:17] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[48:51], s[16:17] offset:64
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[52:55], s[16:17] offset:80
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[40:43], s[16:17] offset:32
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[44:47], s[16:17] offset:48
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(4)
+; CHECK-NEXT: global_store_dwordx4 v0, v[26:29], s[16:17] offset:96
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[30:33], s[16:17] offset:112
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[18:21], s[16:17] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[22:25], s[16:17] offset:80
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[10:13], s[16:17] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[14:17], s[16:17] offset:48
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[2:5], s[16:17]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: global_store_dwordx4 v0, v[6:9], s[16:17] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: global_store_dwordx4 v0, a[32:35], s[16:17]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, a[36:39], s[16:17] offset:16
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, v[8:11], s[16:17]
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v0, v[12:15], s[16:17]
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; CHECK-NEXT: global_store_dwordx4 v0, a[36:39], s[16:17]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
%mai = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 %x, i32 %y, <4 x i32> %arg, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 3bc67562012e5..8f2744eacf543 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -574,8 +574,16 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 3, killed [[COPY1]](s32), implicit $exec
; SI-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR killed %11, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s64) from %ir.idx, addrspace 1)
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[GLOBAL_LOAD_DWORDX2_SADDR]], 16, 0, implicit $exec :: (invariant load (s128) from %ir.3 + 16, addrspace 4)
+ ; SI-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_]].sub3
+ ; SI-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_]].sub2
+ ; SI-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_]].sub1
+ ; SI-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[GLOBAL_LOAD_DWORDX4_]].sub0
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_1:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[GLOBAL_LOAD_DWORDX2_SADDR]], 0, 0, implicit $exec :: (invariant load (s128) from %ir.3, align 32, addrspace 4)
- ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE killed [[GLOBAL_LOAD_DWORDX4_1]].sub0, %subreg.sub0, [[GLOBAL_LOAD_DWORDX4_1]].sub1, %subreg.sub1, [[GLOBAL_LOAD_DWORDX4_1]].sub2, %subreg.sub2, [[GLOBAL_LOAD_DWORDX4_1]].sub3, %subreg.sub3, killed [[GLOBAL_LOAD_DWORDX4_]].sub0, %subreg.sub4, [[GLOBAL_LOAD_DWORDX4_]].sub1, %subreg.sub5, [[GLOBAL_LOAD_DWORDX4_]].sub2, %subreg.sub6, [[GLOBAL_LOAD_DWORDX4_]].sub3, %subreg.sub7
+ ; SI-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_1]].sub3
+ ; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_1]].sub2
+ ; SI-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[GLOBAL_LOAD_DWORDX4_1]].sub1
+ ; SI-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[GLOBAL_LOAD_DWORDX4_1]].sub0
+ ; SI-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1, killed [[COPY7]], %subreg.sub2, killed [[COPY6]], %subreg.sub3, killed [[COPY5]], %subreg.sub4, killed [[COPY4]], %subreg.sub5, killed [[COPY3]], %subreg.sub6, killed [[COPY2]], %subreg.sub7
; SI-NEXT: [[GLOBAL_LOAD_DWORDX4_2:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 killed [[GLOBAL_LOAD_DWORDX2_SADDR]], 48, 0, implicit $exec :: (invariant load (s128) from %ir.add.ptr.i, addrspace 4)
; SI-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; SI-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
index fbacb53f2858e..418cd75f24dea 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
@@ -295,48 +295,55 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b32 s0, s33
; CHECK-NEXT: s_mov_b32 s33, s32
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
-; CHECK-NEXT: scratch_store_b32 off, v93, s33 offset:404 ; 4-byte Folded Spill
+; CHECK-NEXT: scratch_store_b32 off, v47, s33 offset:432 ; 4-byte Folded Spill
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_mov_b32 exec_lo, s1
; CHECK-NEXT: s_clause 0x1f ; 128-byte Folded Spill
-; CHECK-NEXT: scratch_store_b32 off, v40, s33 offset:144
-; CHECK-NEXT: scratch_store_b32 off, v41, s33 offset:140
-; CHECK-NEXT: scratch_store_b32 off, v42, s33 offset:136
-; CHECK-NEXT: scratch_store_b32 off, v43, s33 offset:132
-; CHECK-NEXT: scratch_store_b32 off, v44, s33 offset:128
-; CHECK-NEXT: scratch_store_b32 off, v45, s33 offset:124
-; CHECK-NEXT: scratch_store_b32 off, v46, s33 offset:120
-; CHECK-NEXT: scratch_store_b32 off, v47, s33 offset:116
-; CHECK-NEXT: scratch_store_b32 off, v56, s33 offset:112
-; CHECK-NEXT: scratch_store_b32 off, v57, s33 offset:108
-; CHECK-NEXT: scratch_store_b32 off, v58, s33 offset:104
-; CHECK-NEXT: scratch_store_b32 off, v59, s33 offset:100
-; CHECK-NEXT: scratch_store_b32 off, v60, s33 offset:96
-; CHECK-NEXT: scratch_store_b32 off, v61, s33 offset:92
-; CHECK-NEXT: scratch_store_b32 off, v62, s33 offset:88
-; CHECK-NEXT: scratch_store_b32 off, v63, s33 offset:84
-; CHECK-NEXT: scratch_store_b32 off, v72, s33 offset:80
-; CHECK-NEXT: scratch_store_b32 off, v73, s33 offset:76
-; CHECK-NEXT: scratch_store_b32 off, v74, s33 offset:72
-; CHECK-NEXT: scratch_store_b32 off, v75, s33 offset:68
-; CHECK-NEXT: scratch_store_b32 off, v76, s33 offset:64
-; CHECK-NEXT: scratch_store_b32 off, v77, s33 offset:60
-; CHECK-NEXT: scratch_store_b32 off, v78, s33 offset:56
-; CHECK-NEXT: scratch_store_b32 off, v79, s33 offset:52
-; CHECK-NEXT: scratch_store_b32 off, v88, s33 offset:48
-; CHECK-NEXT: scratch_store_b32 off, v89, s33 offset:44
-; CHECK-NEXT: scratch_store_b32 off, v90, s33 offset:40
-; CHECK-NEXT: scratch_store_b32 off, v91, s33 offset:36
-; CHECK-NEXT: scratch_store_b32 off, v92, s33 offset:32
-; CHECK-NEXT: scratch_store_b32 off, v104, s33 offset:28
-; CHECK-NEXT: scratch_store_b32 off, v105, s33 offset:24
-; CHECK-NEXT: scratch_store_b32 off, v106, s33 offset:20
-; CHECK-NEXT: s_clause 0x4 ; 20-byte Folded Spill
-; CHECK-NEXT: scratch_store_b32 off, v107, s33 offset:16
-; CHECK-NEXT: scratch_store_b32 off, v108, s33 offset:12
-; CHECK-NEXT: scratch_store_b32 off, v109, s33 offset:8
-; CHECK-NEXT: scratch_store_b32 off, v110, s33 offset:4
-; CHECK-NEXT: scratch_store_b32 off, v111, s33
+; CHECK-NEXT: scratch_store_b32 off, v40, s33 offset:172
+; CHECK-NEXT: scratch_store_b32 off, v41, s33 offset:168
+; CHECK-NEXT: scratch_store_b32 off, v42, s33 offset:164
+; CHECK-NEXT: scratch_store_b32 off, v43, s33 offset:160
+; CHECK-NEXT: scratch_store_b32 off, v44, s33 offset:156
+; CHECK-NEXT: scratch_store_b32 off, v45, s33 offset:152
+; CHECK-NEXT: scratch_store_b32 off, v46, s33 offset:148
+; CHECK-NEXT: scratch_store_b32 off, v56, s33 offset:144
+; CHECK-NEXT: scratch_store_b32 off, v57, s33 offset:140
+; CHECK-NEXT: scratch_store_b32 off, v58, s33 offset:136
+; CHECK-NEXT: scratch_store_b32 off, v59, s33 offset:132
+; CHECK-NEXT: scratch_store_b32 off, v60, s33 offset:128
+; CHECK-NEXT: scratch_store_b32 off, v61, s33 offset:124
+; CHECK-NEXT: scratch_store_b32 off, v62, s33 offset:120
+; CHECK-NEXT: scratch_store_b32 off, v63, s33 offset:116
+; CHECK-NEXT: scratch_store_b32 off, v72, s33 offset:112
+; CHECK-NEXT: scratch_store_b32 off, v73, s33 offset:108
+; CHECK-NEXT: scratch_store_b32 off, v74, s33 offset:104
+; CHECK-NEXT: scratch_store_b32 off, v75, s33 offset:100
+; CHECK-NEXT: scratch_store_b32 off, v76, s33 offset:96
+; CHECK-NEXT: scratch_store_b32 off, v77, s33 offset:92
+; CHECK-NEXT: scratch_store_b32 off, v78, s33 offset:88
+; CHECK-NEXT: scratch_store_b32 off, v79, s33 offset:84
+; CHECK-NEXT: scratch_store_b32 off, v88, s33 offset:80
+; CHECK-NEXT: scratch_store_b32 off, v89, s33 offset:76
+; CHECK-NEXT: scratch_store_b32 off, v90, s33 offset:72
+; CHECK-NEXT: scratch_store_b32 off, v91, s33 offset:68
+; CHECK-NEXT: scratch_store_b32 off, v92, s33 offset:64
+; CHECK-NEXT: scratch_store_b32 off, v104, s33 offset:60
+; CHECK-NEXT: scratch_store_b32 off, v105, s33 offset:56
+; CHECK-NEXT: scratch_store_b32 off, v106, s33 offset:52
+; CHECK-NEXT: scratch_store_b32 off, v107, s33 offset:48
+; CHECK-NEXT: s_clause 0xb ; 48-byte Folded Spill
+; CHECK-NEXT: scratch_store_b32 off, v108, s33 offset:44
+; CHECK-NEXT: scratch_store_b32 off, v109, s33 offset:40
+; CHECK-NEXT: scratch_store_b32 off, v110, s33 offset:36
+; CHECK-NEXT: scratch_store_b32 off, v111, s33 offset:32
+; CHECK-NEXT: scratch_store_b32 off, v120, s33 offset:28
+; CHECK-NEXT: scratch_store_b32 off, v121, s33 offset:24
+; CHECK-NEXT: scratch_store_b32 off, v122, s33 offset:20
+; CHECK-NEXT: scratch_store_b32 off, v123, s33 offset:16
+; CHECK-NEXT: scratch_store_b32 off, v124, s33 offset:12
+; CHECK-NEXT: scratch_store_b32 off, v125, s33 offset:8
+; CHECK-NEXT: scratch_store_b32 off, v126, s33 offset:4
+; CHECK-NEXT: scratch_store_b32 off, v127, s33
; CHECK-NEXT: v_dual_mov_b32 v92, v31 :: v_dual_mov_b32 v1, 0
; CHECK-NEXT: v_dual_mov_b32 v0, 0x60 :: v_dual_mov_b32 v3, 0
; CHECK-NEXT: v_dual_mov_b32 v2, 0x50 :: v_dual_mov_b32 v5, 0
@@ -363,8 +370,8 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: global_load_b128 v[76:79], v[4:5], off
; CHECK-NEXT: global_load_b128 v[88:91], v[6:7], off
-; CHECK-NEXT: v_writelane_b32 v93, s0, 14
-; CHECK-NEXT: s_addk_co_i32 s32, 0x1a0
+; CHECK-NEXT: v_writelane_b32 v47, s0, 14
+; CHECK-NEXT: s_addk_co_i32 s32, 0x1c0
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_sext_i32_i16 s1, s1
@@ -372,73 +379,73 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
; CHECK-NEXT: s_wait_loadcnt 0x5
-; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:148 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:176 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x4
-; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:164 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:192 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x3
-; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:180 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:208 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x2
; CHECK-NEXT: s_clause 0x4 ; 80-byte Folded Spill
-; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:196
-; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:212
-; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:228
-; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:244
-; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:260
+; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:224
+; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:240
+; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:256
+; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:272
+; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:288
; CHECK-NEXT: s_clause 0x4
; CHECK-NEXT: global_load_b128 v[10:13], v[8:9], off offset:48
; CHECK-NEXT: global_load_b128 v[14:17], v[8:9], off offset:64
; CHECK-NEXT: global_load_b128 v[18:21], v[8:9], off offset:80
; CHECK-NEXT: global_load_b128 v[22:25], v[8:9], off offset:96
; CHECK-NEXT: global_load_b128 v[26:29], v[8:9], off offset:112
-; CHECK-NEXT: v_writelane_b32 v93, s30, 0
+; CHECK-NEXT: v_writelane_b32 v47, s30, 0
; CHECK-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; CHECK-NEXT: s_wait_loadcnt 0x4
-; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:276 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:304 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x3
-; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:292 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:320 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:308 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:336 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x1
-; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:324 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:352 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x0
; CHECK-NEXT: s_clause 0x3 ; 64-byte Folded Spill
-; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:340
-; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:356
-; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:372
-; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:388
-; CHECK-NEXT: v_writelane_b32 v93, s31, 1
-; CHECK-NEXT: v_writelane_b32 v93, s34, 2
-; CHECK-NEXT: v_writelane_b32 v93, s35, 3
+; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:368
+; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:384
+; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:400
+; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:416
+; CHECK-NEXT: v_writelane_b32 v47, s31, 1
+; CHECK-NEXT: v_writelane_b32 v47, s34, 2
+; CHECK-NEXT: v_writelane_b32 v47, s35, 3
; CHECK-NEXT: s_mov_b64 s[34:35], s[10:11]
-; CHECK-NEXT: v_writelane_b32 v93, s36, 4
-; CHECK-NEXT: v_writelane_b32 v93, s37, 5
+; CHECK-NEXT: v_writelane_b32 v47, s36, 4
+; CHECK-NEXT: v_writelane_b32 v47, s37, 5
; CHECK-NEXT: s_mov_b64 s[36:37], s[8:9]
-; CHECK-NEXT: v_writelane_b32 v93, s38, 6
-; CHECK-NEXT: v_writelane_b32 v93, s39, 7
+; CHECK-NEXT: v_writelane_b32 v47, s38, 6
+; CHECK-NEXT: v_writelane_b32 v47, s39, 7
; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
-; CHECK-NEXT: v_writelane_b32 v93, s48, 8
-; CHECK-NEXT: v_writelane_b32 v93, s49, 9
+; CHECK-NEXT: v_writelane_b32 v47, s48, 8
+; CHECK-NEXT: v_writelane_b32 v47, s49, 9
; CHECK-NEXT: s_mov_b64 s[48:49], s[4:5]
-; CHECK-NEXT: v_writelane_b32 v93, s50, 10
+; CHECK-NEXT: v_writelane_b32 v47, s50, 10
; CHECK-NEXT: s_mov_b32 s50, s15
-; CHECK-NEXT: v_writelane_b32 v93, s51, 11
+; CHECK-NEXT: v_writelane_b32 v47, s51, 11
; CHECK-NEXT: s_mov_b32 s51, s14
-; CHECK-NEXT: v_writelane_b32 v93, s52, 12
+; CHECK-NEXT: v_writelane_b32 v47, s52, 12
; CHECK-NEXT: s_mov_b32 s52, s13
-; CHECK-NEXT: v_writelane_b32 v93, s53, 13
+; CHECK-NEXT: v_writelane_b32 v47, s53, 13
; CHECK-NEXT: s_mov_b32 s53, s12
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
; CHECK-NEXT: s_clause 0x7 ; 128-byte Folded Reload
-; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:276 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:292 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:308 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:324 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:340 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:356 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:372 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:388 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:304 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:320 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:336 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:352 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:368 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:384 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:400 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:416 th:TH_LOAD_LU
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_sext_i32_i16 s1, s1
@@ -454,121 +461,137 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b32 s13, s52
; CHECK-NEXT: s_mov_b32 s14, s51
; CHECK-NEXT: s_mov_b32 s15, s50
+; CHECK-NEXT: s_wait_loadcnt 0x4
+; CHECK-NEXT: v_dual_mov_b32 v127, v15 :: v_dual_mov_b32 v126, v14
+; CHECK-NEXT: s_wait_loadcnt 0x3
+; CHECK-NEXT: v_dual_mov_b32 v131, v19 :: v_dual_mov_b32 v130, v18
+; CHECK-NEXT: v_dual_mov_b32 v125, v13 :: v_dual_mov_b32 v124, v12
+; CHECK-NEXT: v_dual_mov_b32 v123, v11 :: v_dual_mov_b32 v122, v10
+; CHECK-NEXT: v_dual_mov_b32 v121, v9 :: v_dual_mov_b32 v120, v8
+; CHECK-NEXT: v_dual_mov_b32 v119, v7 :: v_dual_mov_b32 v118, v6
+; CHECK-NEXT: v_dual_mov_b32 v117, v5 :: v_dual_mov_b32 v116, v4
+; CHECK-NEXT: v_dual_mov_b32 v115, v3 :: v_dual_mov_b32 v114, v2
+; CHECK-NEXT: v_dual_mov_b32 v113, v1 :: v_dual_mov_b32 v112, v0
+; CHECK-NEXT: v_dual_mov_b32 v129, v17 :: v_dual_mov_b32 v128, v16
+; CHECK-NEXT: s_clause 0x4 ; 80-byte Folded Reload
+; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:176 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:192 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:208 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:224 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:240 th:TH_LOAD_LU
+; CHECK-NEXT: s_wait_loadcnt 0x7
+; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:256 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x7
+; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:272 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x7
+; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:288 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: v_dual_mov_b32 v45, v113 :: v_dual_mov_b32 v44, v112
+; CHECK-NEXT: v_mov_b32_e32 v46, v114
+; CHECK-NEXT: s_clause 0x4
+; CHECK-NEXT: scratch_store_b32 off, v131, s32 offset:128
+; CHECK-NEXT: scratch_store_b128 off, v[127:130], s32 offset:112
+; CHECK-NEXT: scratch_store_b128 off, v[123:126], s32 offset:96
+; CHECK-NEXT: scratch_store_b128 off, v[119:122], s32 offset:80
+; CHECK-NEXT: scratch_store_b128 off, v[115:118], s32 offset:64
; CHECK-NEXT: s_wait_loadcnt 0x3
-; CHECK-NEXT: scratch_store_b32 off, v19, s32 offset:128
-; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_load_b128 v[19:22], off, s33 offset:148 th:TH_LOAD_LU ; 16-byte Folded Reload
-; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_load_b128 v[23:26], off, s33 offset:164 th:TH_LOAD_LU ; 16-byte Folded Reload
-; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: s_clause 0x5 ; 96-byte Folded Reload
-; CHECK-NEXT: scratch_load_b128 v[27:30], off, s33 offset:180 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[31:34], off, s33 offset:196 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[35:38], off, s33 offset:212 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[39:42], off, s33 offset:228 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[43:46], off, s33 offset:244 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[47:50], off, s33 offset:260 th:TH_LOAD_LU
-; CHECK-NEXT: s_clause 0x3
-; CHECK-NEXT: scratch_store_b128 off, v[15:18], s32 offset:112
-; CHECK-NEXT: scratch_store_b128 off, v[11:14], s32 offset:96
-; CHECK-NEXT: scratch_store_b128 off, v[7:10], s32 offset:80
-; CHECK-NEXT: scratch_store_b128 off, v[3:6], s32 offset:64
-; CHECK-NEXT: v_dual_mov_b32 v4, v88 :: v_dual_mov_b32 v5, v89
-; CHECK-NEXT: v_dual_mov_b32 v6, v90 :: v_dual_mov_b32 v7, v91
-; CHECK-NEXT: v_dual_mov_b32 v8, v76 :: v_dual_mov_b32 v9, v77
-; CHECK-NEXT: v_dual_mov_b32 v10, v78 :: v_dual_mov_b32 v11, v79
-; CHECK-NEXT: v_dual_mov_b32 v12, v60 :: v_dual_mov_b32 v13, v61
-; CHECK-NEXT: v_dual_mov_b32 v14, v62 :: v_dual_mov_b32 v15, v63
; CHECK-NEXT: v_dual_mov_b32 v16, v108 :: v_dual_mov_b32 v17, v109
-; CHECK-NEXT: v_mov_b32_e32 v18, v110
+; CHECK-NEXT: v_dual_mov_b32 v18, v110 :: v_dual_mov_b32 v19, v111
+; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: v_dual_mov_b32 v20, v104 :: v_dual_mov_b32 v21, v105
+; CHECK-NEXT: v_dual_mov_b32 v22, v106 :: v_dual_mov_b32 v23, v107
; CHECK-NEXT: s_wait_loadcnt 0x1
-; CHECK-NEXT: v_dual_mov_b32 v44, v0 :: v_dual_mov_b32 v45, v1
-; CHECK-NEXT: v_mov_b32_e32 v46, v2
-; CHECK-NEXT: v_dual_mov_b32 v0, v72 :: v_dual_mov_b32 v1, v73
-; CHECK-NEXT: v_dual_mov_b32 v2, v74 :: v_dual_mov_b32 v3, v75
-; CHECK-NEXT: v_mov_b32_e32 v43, v34
-; CHECK-NEXT: v_dual_mov_b32 v42, v33 :: v_dual_mov_b32 v41, v32
-; CHECK-NEXT: v_dual_mov_b32 v40, v31 :: v_dual_mov_b32 v39, v30
-; CHECK-NEXT: v_dual_mov_b32 v38, v29 :: v_dual_mov_b32 v37, v28
-; CHECK-NEXT: v_dual_mov_b32 v36, v27 :: v_dual_mov_b32 v35, v26
-; CHECK-NEXT: v_mov_b32_e32 v34, v25
-; CHECK-NEXT: v_mov_b32_e32 v33, v24
-; CHECK-NEXT: v_mov_b32_e32 v32, v23
-; CHECK-NEXT: v_mov_b32_e32 v31, v22
-; CHECK-NEXT: v_mov_b32_e32 v30, v21
-; CHECK-NEXT: v_mov_b32_e32 v29, v20
-; CHECK-NEXT: v_mov_b32_e32 v28, v19
+; CHECK-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; CHECK-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: v_dual_mov_b32 v43, v15 :: v_dual_mov_b32 v42, v14
+; CHECK-NEXT: v_dual_mov_b32 v41, v13 :: v_dual_mov_b32 v40, v12
+; CHECK-NEXT: v_dual_mov_b32 v39, v11 :: v_dual_mov_b32 v38, v10
+; CHECK-NEXT: v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v36, v8
+; CHECK-NEXT: v_dual_mov_b32 v35, v7 :: v_dual_mov_b32 v34, v6
+; CHECK-NEXT: v_dual_mov_b32 v33, v5 :: v_dual_mov_b32 v32, v4
+; CHECK-NEXT: v_dual_mov_b32 v31, v3 :: v_dual_mov_b32 v30, v2
+; CHECK-NEXT: v_dual_mov_b32 v29, v1 :: v_dual_mov_b32 v28, v0
; CHECK-NEXT: s_clause 0x3
; CHECK-NEXT: scratch_store_b128 off, v[43:46], s32 offset:48
; CHECK-NEXT: scratch_store_b128 off, v[39:42], s32 offset:32
; CHECK-NEXT: scratch_store_b128 off, v[35:38], s32 offset:16
; CHECK-NEXT: scratch_store_b128 off, v[31:34], s32
-; CHECK-NEXT: v_mov_b32_e32 v31, v92
-; CHECK-NEXT: v_dual_mov_b32 v19, v111 :: v_dual_mov_b32 v20, v104
-; CHECK-NEXT: v_dual_mov_b32 v21, v105 :: v_dual_mov_b32 v22, v106
-; CHECK-NEXT: v_dual_mov_b32 v23, v107 :: v_dual_mov_b32 v24, v56
-; CHECK-NEXT: v_dual_mov_b32 v25, v57 :: v_dual_mov_b32 v26, v58
-; CHECK-NEXT: v_mov_b32_e32 v27, v59
+; CHECK-NEXT: v_dual_mov_b32 v31, v92 :: v_dual_mov_b32 v0, v72
+; CHECK-NEXT: v_dual_mov_b32 v1, v73 :: v_dual_mov_b32 v2, v74
+; CHECK-NEXT: v_dual_mov_b32 v3, v75 :: v_dual_mov_b32 v4, v88
+; CHECK-NEXT: v_dual_mov_b32 v5, v89 :: v_dual_mov_b32 v6, v90
+; CHECK-NEXT: v_dual_mov_b32 v7, v91 :: v_dual_mov_b32 v8, v76
+; CHECK-NEXT: v_dual_mov_b32 v9, v77 :: v_dual_mov_b32 v10, v78
+; CHECK-NEXT: v_dual_mov_b32 v11, v79 :: v_dual_mov_b32 v12, v60
+; CHECK-NEXT: v_dual_mov_b32 v13, v61 :: v_dual_mov_b32 v14, v62
+; CHECK-NEXT: v_mov_b32_e32 v15, v63
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
; CHECK-NEXT: s_clause 0x1f ; 128-byte Folded Reload
-; CHECK-NEXT: scratch_load_b32 v111, off, s33
-; CHECK-NEXT: scratch_load_b32 v110, off, s33 offset:4
-; CHECK-NEXT: scratch_load_b32 v109, off, s33 offset:8
-; CHECK-NEXT: scratch_load_b32 v108, off, s33 offset:12
-; CHECK-NEXT: scratch_load_b32 v107, off, s33 offset:16
-; CHECK-NEXT: scratch_load_b32 v106, off, s33 offset:20
-; CHECK-NEXT: scratch_load_b32 v105, off, s33 offset:24
-; CHECK-NEXT: scratch_load_b32 v104, off, s33 offset:28
-; CHECK-NEXT: scratch_load_b32 v92, off, s33 offset:32
-; CHECK-NEXT: scratch_load_b32 v91, off, s33 offset:36
-; CHECK-NEXT: scratch_load_b32 v90, off, s33 offset:40
-; CHECK-NEXT: scratch_load_b32 v89, off, s33 offset:44
-; CHECK-NEXT: scratch_load_b32 v88, off, s33 offset:48
-; CHECK-NEXT: scratch_load_b32 v79, off, s33 offset:52
-; CHECK-NEXT: scratch_load_b32 v78, off, s33 offset:56
-; CHECK-NEXT: scratch_load_b32 v77, off, s33 offset:60
-; CHECK-NEXT: scratch_load_b32 v76, off, s33 offset:64
-; CHECK-NEXT: scratch_load_b32 v75, off, s33 offset:68
-; CHECK-NEXT: scratch_load_b32 v74, off, s33 offset:72
-; CHECK-NEXT: scratch_load_b32 v73, off, s33 offset:76
-; CHECK-NEXT: scratch_load_b32 v72, off, s33 offset:80
-; CHECK-NEXT: scratch_load_b32 v63, off, s33 offset:84
-; CHECK-NEXT: scratch_load_b32 v62, off, s33 offset:88
-; CHECK-NEXT: scratch_load_b32 v61, off, s33 offset:92
-; CHECK-NEXT: scratch_load_b32 v60, off, s33 offset:96
-; CHECK-NEXT: scratch_load_b32 v59, off, s33 offset:100
-; CHECK-NEXT: scratch_load_b32 v58, off, s33 offset:104
-; CHECK-NEXT: scratch_load_b32 v57, off, s33 offset:108
-; CHECK-NEXT: scratch_load_b32 v56, off, s33 offset:112
-; CHECK-NEXT: scratch_load_b32 v47, off, s33 offset:116
-; CHECK-NEXT: scratch_load_b32 v46, off, s33 offset:120
-; CHECK-NEXT: scratch_load_b32 v45, off, s33 offset:124
-; CHECK-NEXT: s_clause 0x4 ; 20-byte Folded Reload
-; CHECK-NEXT: scratch_load_b32 v44, off, s33 offset:128
-; CHECK-NEXT: scratch_load_b32 v43, off, s33 offset:132
-; CHECK-NEXT: scratch_load_b32 v42, off, s33 offset:136
-; CHECK-NEXT: scratch_load_b32 v41, off, s33 offset:140
-; CHECK-NEXT: scratch_load_b32 v40, off, s33 offset:144
-; CHECK-NEXT: v_readlane_b32 s53, v93, 13
-; CHECK-NEXT: v_readlane_b32 s52, v93, 12
-; CHECK-NEXT: v_readlane_b32 s51, v93, 11
-; CHECK-NEXT: v_readlane_b32 s50, v93, 10
-; CHECK-NEXT: v_readlane_b32 s49, v93, 9
-; CHECK-NEXT: v_readlane_b32 s48, v93, 8
-; CHECK-NEXT: v_readlane_b32 s39, v93, 7
-; CHECK-NEXT: v_readlane_b32 s38, v93, 6
-; CHECK-NEXT: v_readlane_b32 s37, v93, 5
-; CHECK-NEXT: v_readlane_b32 s36, v93, 4
-; CHECK-NEXT: v_readlane_b32 s35, v93, 3
-; CHECK-NEXT: v_readlane_b32 s34, v93, 2
-; CHECK-NEXT: v_readlane_b32 s31, v93, 1
-; CHECK-NEXT: v_readlane_b32 s30, v93, 0
+; CHECK-NEXT: scratch_load_b32 v127, off, s33
+; CHECK-NEXT: scratch_load_b32 v126, off, s33 offset:4
+; CHECK-NEXT: scratch_load_b32 v125, off, s33 offset:8
+; CHECK-NEXT: scratch_load_b32 v124, off, s33 offset:12
+; CHECK-NEXT: scratch_load_b32 v123, off, s33 offset:16
+; CHECK-NEXT: scratch_load_b32 v122, off, s33 offset:20
+; CHECK-NEXT: scratch_load_b32 v121, off, s33 offset:24
+; CHECK-NEXT: scratch_load_b32 v120, off, s33 offset:28
+; CHECK-NEXT: scratch_load_b32 v111, off, s33 offset:32
+; CHECK-NEXT: scratch_load_b32 v110, off, s33 offset:36
+; CHECK-NEXT: scratch_load_b32 v109, off, s33 offset:40
+; CHECK-NEXT: scratch_load_b32 v108, off, s33 offset:44
+; CHECK-NEXT: scratch_load_b32 v107, off, s33 offset:48
+; CHECK-NEXT: scratch_load_b32 v106, off, s33 offset:52
+; CHECK-NEXT: scratch_load_b32 v105, off, s33 offset:56
+; CHECK-NEXT: scratch_load_b32 v104, off, s33 offset:60
+; CHECK-NEXT: scratch_load_b32 v92, off, s33 offset:64
+; CHECK-NEXT: scratch_load_b32 v91, off, s33 offset:68
+; CHECK-NEXT: scratch_load_b32 v90, off, s33 offset:72
+; CHECK-NEXT: scratch_load_b32 v89, off, s33 offset:76
+; CHECK-NEXT: scratch_load_b32 v88, off, s33 offset:80
+; CHECK-NEXT: scratch_load_b32 v79, off, s33 offset:84
+; CHECK-NEXT: scratch_load_b32 v78, off, s33 offset:88
+; CHECK-NEXT: scratch_load_b32 v77, off, s33 offset:92
+; CHECK-NEXT: scratch_load_b32 v76, off, s33 offset:96
+; CHECK-NEXT: scratch_load_b32 v75, off, s33 offset:100
+; CHECK-NEXT: scratch_load_b32 v74, off, s33 offset:104
+; CHECK-NEXT: scratch_load_b32 v73, off, s33 offset:108
+; CHECK-NEXT: scratch_load_b32 v72, off, s33 offset:112
+; CHECK-NEXT: scratch_load_b32 v63, off, s33 offset:116
+; CHECK-NEXT: scratch_load_b32 v62, off, s33 offset:120
+; CHECK-NEXT: scratch_load_b32 v61, off, s33 offset:124
+; CHECK-NEXT: s_clause 0xb ; 48-byte Folded Reload
+; CHECK-NEXT: scratch_load_b32 v60, off, s33 offset:128
+; CHECK-NEXT: scratch_load_b32 v59, off, s33 offset:132
+; CHECK-NEXT: scratch_load_b32 v58, off, s33 offset:136
+; CHECK-NEXT: scratch_load_b32 v57, off, s33 offset:140
+; CHECK-NEXT: scratch_load_b32 v56, off, s33 offset:144
+; CHECK-NEXT: scratch_load_b32 v46, off, s33 offset:148
+; CHECK-NEXT: scratch_load_b32 v45, off, s33 offset:152
+; CHECK-NEXT: scratch_load_b32 v44, off, s33 offset:156
+; CHECK-NEXT: scratch_load_b32 v43, off, s33 offset:160
+; CHECK-NEXT: scratch_load_b32 v42, off, s33 offset:164
+; CHECK-NEXT: scratch_load_b32 v41, off, s33 offset:168
+; CHECK-NEXT: scratch_load_b32 v40, off, s33 offset:172
+; CHECK-NEXT: v_readlane_b32 s53, v47, 13
+; CHECK-NEXT: v_readlane_b32 s52, v47, 12
+; CHECK-NEXT: v_readlane_b32 s51, v47, 11
+; CHECK-NEXT: v_readlane_b32 s50, v47, 10
+; CHECK-NEXT: v_readlane_b32 s49, v47, 9
+; CHECK-NEXT: v_readlane_b32 s48, v47, 8
+; CHECK-NEXT: v_readlane_b32 s39, v47, 7
+; CHECK-NEXT: v_readlane_b32 s38, v47, 6
+; CHECK-NEXT: v_readlane_b32 s37, v47, 5
+; CHECK-NEXT: v_readlane_b32 s36, v47, 4
+; CHECK-NEXT: v_readlane_b32 s35, v47, 3
+; CHECK-NEXT: v_readlane_b32 s34, v47, 2
+; CHECK-NEXT: v_readlane_b32 s31, v47, 1
+; CHECK-NEXT: v_readlane_b32 s30, v47, 0
; CHECK-NEXT: s_mov_b32 s32, s33
-; CHECK-NEXT: v_readlane_b32 s0, v93, 14
+; CHECK-NEXT: v_readlane_b32 s0, v47, 14
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
-; CHECK-NEXT: scratch_load_b32 v93, off, s33 offset:404 ; 4-byte Folded Reload
+; CHECK-NEXT: scratch_load_b32 v47, off, s33 offset:432 ; 4-byte Folded Reload
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_mov_b32 exec_lo, s1
; CHECK-NEXT: s_mov_b32 s33, s0
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index 20789232b1f25..3263642d51282 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -218,21 +218,21 @@ define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1
; GFX942-NEXT: v_lshlrev_b32_e32 v1, 5, v0
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB5_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 5, v0
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3]
; GFX942-NEXT: .LBB5_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:16
+; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:16
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7]
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -260,69 +260,69 @@ define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:240
-; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:224
-; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[0:1] offset:208
-; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[0:1] offset:192
-; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[0:1] offset:176
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:160
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:144
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:128
-; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1] offset:112
-; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[0:1] offset:96
-; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[0:1] offset:80
-; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[0:1] offset:64
-; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[0:1] offset:48
-; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:32
-; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v1, s[0:1] offset:240
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v1, s[0:1] offset:224
+; GFX942-NEXT: global_load_dwordx4 v[10:13], v1, s[0:1] offset:208
+; GFX942-NEXT: global_load_dwordx4 v[14:17], v1, s[0:1] offset:192
+; GFX942-NEXT: global_load_dwordx4 v[18:21], v1, s[0:1] offset:176
+; GFX942-NEXT: global_load_dwordx4 v[22:25], v1, s[0:1] offset:160
+; GFX942-NEXT: global_load_dwordx4 v[26:29], v1, s[0:1] offset:144
+; GFX942-NEXT: global_load_dwordx4 v[30:33], v1, s[0:1] offset:128
+; GFX942-NEXT: global_load_dwordx4 v[34:37], v1, s[0:1] offset:112
+; GFX942-NEXT: global_load_dwordx4 v[38:41], v1, s[0:1] offset:96
+; GFX942-NEXT: global_load_dwordx4 v[42:45], v1, s[0:1] offset:80
+; GFX942-NEXT: global_load_dwordx4 v[46:49], v1, s[0:1] offset:64
+; GFX942-NEXT: global_load_dwordx4 v[50:53], v1, s[0:1] offset:48
+; GFX942-NEXT: global_load_dwordx4 v[54:57], v1, s[0:1] offset:32
+; GFX942-NEXT: global_load_dwordx4 v[58:61], v1, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 a[0:3], v1, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB6_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX942-NEXT: global_load_dwordx4 v[30:33], v0, s[2:3] offset:240
-; GFX942-NEXT: global_load_dwordx4 v[26:29], v0, s[2:3] offset:224
-; GFX942-NEXT: global_load_dwordx4 v[22:25], v0, s[2:3] offset:208
-; GFX942-NEXT: global_load_dwordx4 v[18:21], v0, s[2:3] offset:192
-; GFX942-NEXT: global_load_dwordx4 v[14:17], v0, s[2:3] offset:176
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v0, s[2:3] offset:160
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:144
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:128
-; GFX942-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3] offset:112
-; GFX942-NEXT: global_load_dwordx4 v[58:61], v0, s[2:3] offset:96
-; GFX942-NEXT: global_load_dwordx4 v[54:57], v0, s[2:3] offset:80
-; GFX942-NEXT: global_load_dwordx4 v[50:53], v0, s[2:3] offset:64
-; GFX942-NEXT: global_load_dwordx4 v[46:49], v0, s[2:3] offset:48
-; GFX942-NEXT: global_load_dwordx4 v[42:45], v0, s[2:3] offset:32
-; GFX942-NEXT: global_load_dwordx4 v[38:41], v0, s[2:3] offset:16
-; GFX942-NEXT: global_load_dwordx4 v[34:37], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3] offset:240
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v0, s[2:3] offset:224
+; GFX942-NEXT: global_load_dwordx4 v[10:13], v0, s[2:3] offset:208
+; GFX942-NEXT: global_load_dwordx4 v[14:17], v0, s[2:3] offset:192
+; GFX942-NEXT: global_load_dwordx4 v[18:21], v0, s[2:3] offset:176
+; GFX942-NEXT: global_load_dwordx4 v[22:25], v0, s[2:3] offset:160
+; GFX942-NEXT: global_load_dwordx4 v[26:29], v0, s[2:3] offset:144
+; GFX942-NEXT: global_load_dwordx4 v[30:33], v0, s[2:3] offset:128
+; GFX942-NEXT: global_load_dwordx4 v[34:37], v0, s[2:3] offset:112
+; GFX942-NEXT: global_load_dwordx4 v[38:41], v0, s[2:3] offset:96
+; GFX942-NEXT: global_load_dwordx4 v[42:45], v0, s[2:3] offset:80
+; GFX942-NEXT: global_load_dwordx4 v[46:49], v0, s[2:3] offset:64
+; GFX942-NEXT: global_load_dwordx4 v[50:53], v0, s[2:3] offset:48
+; GFX942-NEXT: global_load_dwordx4 v[54:57], v0, s[2:3] offset:32
+; GFX942-NEXT: global_load_dwordx4 v[58:61], v0, s[2:3] offset:16
+; GFX942-NEXT: global_load_dwordx4 a[0:3], v0, s[2:3]
; GFX942-NEXT: .LBB6_2: ; %bb.2
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] offset:112
+; GFX942-NEXT: global_store_dwordx4 v1, v[34:37], s[6:7] offset:112
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[58:61], s[6:7] offset:96
+; GFX942-NEXT: global_store_dwordx4 v1, v[38:41], s[6:7] offset:96
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[54:57], s[6:7] offset:80
+; GFX942-NEXT: global_store_dwordx4 v1, v[42:45], s[6:7] offset:80
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[50:53], s[6:7] offset:64
+; GFX942-NEXT: global_store_dwordx4 v1, v[46:49], s[6:7] offset:64
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[46:49], s[6:7] offset:48
+; GFX942-NEXT: global_store_dwordx4 v1, v[50:53], s[6:7] offset:48
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[42:45], s[6:7] offset:32
+; GFX942-NEXT: global_store_dwordx4 v1, v[54:57], s[6:7] offset:32
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[38:41], s[6:7] offset:16
+; GFX942-NEXT: global_store_dwordx4 v1, v[58:61], s[6:7] offset:16
; GFX942-NEXT: s_waitcnt vmcnt(7)
-; GFX942-NEXT: global_store_dwordx4 v1, v[34:37], s[6:7]
-; GFX942-NEXT: global_store_dwordx4 v1, v[30:33], s[6:7] offset:240
-; GFX942-NEXT: global_store_dwordx4 v1, v[26:29], s[6:7] offset:224
-; GFX942-NEXT: global_store_dwordx4 v1, v[22:25], s[6:7] offset:208
-; GFX942-NEXT: global_store_dwordx4 v1, v[18:21], s[6:7] offset:192
-; GFX942-NEXT: global_store_dwordx4 v1, v[14:17], s[6:7] offset:176
-; GFX942-NEXT: global_store_dwordx4 v1, v[10:13], s[6:7] offset:160
-; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:144
-; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:128
+; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7]
+; GFX942-NEXT: global_store_dwordx4 v1, v[2:5], s[6:7] offset:240
+; GFX942-NEXT: global_store_dwordx4 v1, v[6:9], s[6:7] offset:224
+; GFX942-NEXT: global_store_dwordx4 v1, v[10:13], s[6:7] offset:208
+; GFX942-NEXT: global_store_dwordx4 v1, v[14:17], s[6:7] offset:192
+; GFX942-NEXT: global_store_dwordx4 v1, v[18:21], s[6:7] offset:176
+; GFX942-NEXT: global_store_dwordx4 v1, v[22:25], s[6:7] offset:160
+; GFX942-NEXT: global_store_dwordx4 v1, v[26:29], s[6:7] offset:144
+; GFX942-NEXT: global_store_dwordx4 v1, v[30:33], s[6:7] offset:128
; GFX942-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
index 88f70cca245f8..fc7d81f19e1a3 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-smrd-loads.ll
@@ -9,13 +9,14 @@ define amdgpu_kernel void @widen_i16_constant_load(ptr addrspace(4) %arg) {
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 4
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: s_or_b32 s4, s1, 4
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -59,14 +60,15 @@ define amdgpu_kernel void @widen_i16_constant_load_zext_i32(ptr addrspace(4) %ar
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s2, s2, 0xffff
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 4
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_and_b32 s1, s1, 0xffff
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: s_or_b32 s4, s1, 4
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -114,14 +116,15 @@ define amdgpu_kernel void @widen_i16_constant_load_sext_i32(ptr addrspace(4) %ar
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_sext_i32_i16 s2, s2
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 4
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_sext_i32_i16 s1, s1
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: s_or_b32 s4, s1, 4
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -167,22 +170,26 @@ define amdgpu_kernel void @widen_i16_constant_load_sext_i32(ptr addrspace(4) %ar
define amdgpu_kernel void @widen_i17_constant_load(ptr addrspace(4) %arg) {
; SI-LABEL: widen_i17_constant_load:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s4, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s7, s[4:5], 0x0
+; SI-NEXT: s_mov_b32 s4, 2
+; SI-NEXT: s_mov_b32 s5, s0
+; SI-NEXT: s_mov_b32 s6, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_add_i32 s4, s4, 34
-; SI-NEXT: s_or_b32 s4, s4, 4
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: s_bfe_u32 s4, s4, 0x10010
+; SI-NEXT: s_add_i32 s7, s7, 34
+; SI-NEXT: s_or_b32 s7, s7, 4
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: s_bfe_u32 s8, s7, 0x10010
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
-; SI-NEXT: s_mov_b64 s[0:1], 2
+; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: widen_i17_constant_load:
@@ -240,7 +247,8 @@ define amdgpu_kernel void @widen_f16_constant_load(ptr addrspace(4) %arg) {
; SI-NEXT: s_load_dword s0, s[0:1], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b32 s0, 0
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_add_f32_e32 v0, 4.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -291,18 +299,19 @@ define amdgpu_kernel void @widen_v2i8_constant_load(ptr addrspace(4) %arg) {
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s2, 0xff00
-; SI-NEXT: s_add_i32 s2, s2, 12
-; SI-NEXT: s_or_b32 s2, s2, 4
-; SI-NEXT: s_and_b32 s2, s2, 0xff
-; SI-NEXT: s_or_b32 s2, s4, s2
-; SI-NEXT: s_addk_i32 s2, 0x2c00
-; SI-NEXT: s_or_b32 s4, s2, 0x300
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_and_b32 s4, s1, 0xff00
+; SI-NEXT: s_add_i32 s1, s1, 12
+; SI-NEXT: s_or_b32 s1, s1, 4
+; SI-NEXT: s_and_b32 s1, s1, 0xff
+; SI-NEXT: s_or_b32 s1, s4, s1
+; SI-NEXT: s_addk_i32 s1, 0x2c00
+; SI-NEXT: s_or_b32 s4, s1, 0x300
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -362,12 +371,14 @@ define amdgpu_kernel void @no_widen_i16_constant_divergent_load(ptr addrspace(4)
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_mov_b64 s[0:1], 0
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s2
+; SI-NEXT: s_mov_b32 s5, s2
+; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v0, vcc, 0x3e7, v0
; SI-NEXT: v_or_b32_e32 v0, 4, v0
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: no_widen_i16_constant_divergent_load:
@@ -432,12 +443,13 @@ define amdgpu_kernel void @widen_i1_constant_load(ptr addrspace(4) %arg) {
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s2, 1
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_and_b32 s4, s1, 1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -478,14 +490,15 @@ define amdgpu_kernel void @widen_i16_zextload_i64_constant_load(ptr addrspace(4)
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s2, s2, 0xffff
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 4
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_and_b32 s1, s1, 0xffff
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: s_or_b32 s4, s1, 4
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -534,14 +547,15 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(ptr addrspace(4) %
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s2, s2, 0xff
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_and_b32 s1, s1, 0xff
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -587,13 +601,13 @@ define amdgpu_kernel void @widen_i16_constant32_load(ptr addrspace(6) %arg) {
; SI-NEXT: s_load_dword s0, s[4:5], 0x9
; SI-NEXT: s_mov_b32 s1, 0
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s0, s[0:1], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 4
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_addk_i32 s0, 0x3e7
+; SI-NEXT: s_or_b32 s4, s0, 4
+; SI-NEXT: s_mov_b32 s0, s1
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -639,13 +653,14 @@ define amdgpu_kernel void @widen_i16_global_invariant_load(ptr addrspace(1) %arg
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_load_dword s1, s[0:1], 0x0
+; SI-NEXT: s_mov_b32 s0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_addk_i32 s2, 0x3e7
-; SI-NEXT: s_or_b32 s4, s2, 1
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_addk_i32 s1, 0x3e7
+; SI-NEXT: s_or_b32 s4, s1, 1
+; SI-NEXT: s_mov_b32 s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
index ecb0c8eb9e0be..eb5b158f13826 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
@@ -12,21 +12,22 @@ define amdgpu_kernel void @widen_vselect_and_mask_v4f64(<4 x double> %arg) #0 {
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_mov_b64 s[4:5], 16
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_mov_b32 s4, 16
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: v_mov_b32_e32 v1, v0
; GCN-NEXT: v_mov_b32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v3, v0
-; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCN-NEXT: v_cmp_u_f64_e64 s[2:3], s[0:1], s[0:1]
-; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[2:3]
+; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[2:3]
; GCN-NEXT: v_cmp_neq_f64_e64 s[0:1], s[0:1], 0
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, -1, v1
+; GCN-NEXT: v_cmp_lt_i32_e32 vcc, -1, v4
; GCN-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GCN-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
-; GCN-NEXT: s_mov_b64 s[4:5], 0
+; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s4, s5
+; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCN-NEXT: s_endpgm
@@ -51,11 +52,11 @@ define amdgpu_kernel void @widen_vselect_and_mask_v4i64(<4 x i64> %arg) #0 {
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], 0
-; GCN-NEXT: s_mov_b32 s10, -1
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_mov_b64 s[8:9], 16
-; GCN-NEXT: s_mov_b32 s11, 0xf000
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_mov_b32 s4, 16
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: v_mov_b32_e32 v2, v1
; GCN-NEXT: v_mov_b32_e32 v3, v1
; GCN-NEXT: v_mov_b32_e32 v4, v1
@@ -65,9 +66,8 @@ define amdgpu_kernel void @widen_vselect_and_mask_v4i64(<4 x i64> %arg) #0 {
; GCN-NEXT: v_cmp_lt_i32_e32 vcc, -1, v0
; GCN-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GCN-NEXT: buffer_store_dwordx4 v[1:4], off, s[8:11], 0
-; GCN-NEXT: s_mov_b32 s6, s10
-; GCN-NEXT: s_mov_b32 s7, s11
+; GCN-NEXT: buffer_store_dwordx4 v[1:4], off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s4, s5
; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GCN-NEXT: s_endpgm
bb:
>From 1219ae0bc839cf920c3cd319f8ec84dfaebb6f9a Mon Sep 17 00:00:00 2001
From: Frederik Harwath <fharwath at amd.com>
Date: Tue, 28 Apr 2026 09:42:32 -0400
Subject: [PATCH 2/2] clang-format
---
llvm/lib/CodeGen/PeepholeOptimizer.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 2a5ac2de9e3bb..4d013f35cbff3 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1189,8 +1189,7 @@ bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter,
RewriteMapTy RewriteMap;
// Try to find a more suitable source. If we failed to do so, or get the
// actual source, move to the next source.
- if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap,
- AcceptSubReg))
+ if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap, AcceptSubReg))
continue;
// Get the new source to rewrite. TODO: Only enable handling of multiple
More information about the llvm-commits
mailing list