[llvm] b6eb594 - [AMDGPU] Reschedule loads in clauses to improve throughput (#102595)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 20:58:18 PDT 2026
Author: Carl Ritson
Date: 2026-08-07T12:58:08+09:00
New Revision: b6eb594ad0186dfb349c2903b899606c21c6b348
URL: https://github.com/llvm/llvm-project/commit/b6eb594ad0186dfb349c2903b899606c21c6b348
DIFF: https://github.com/llvm/llvm-project/commit/b6eb594ad0186dfb349c2903b899606c21c6b348.diff
LOG: [AMDGPU] Reschedule loads in clauses to improve throughput (#102595)
After clauses are formed their internal loads can be reordered to
facilitate some additional opportunities for overlapping computation.
This late stage rescheduling causes no change in register pressure.
Added:
llvm/test/CodeGen/AMDGPU/reschedule-bundle-loads.mir
Modified:
llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
llvm/test/CodeGen/AMDGPU/bf16.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
llvm/test/CodeGen/AMDGPU/div_i128.ll
llvm/test/CodeGen/AMDGPU/ds-alignment.ll
llvm/test/CodeGen/AMDGPU/ds_read2.ll
llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
llvm/test/CodeGen/AMDGPU/freeze.ll
llvm/test/CodeGen/AMDGPU/frem.ll
llvm/test/CodeGen/AMDGPU/function-args.ll
llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
llvm/test/CodeGen/AMDGPU/idot4u.ll
llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
llvm/test/CodeGen/AMDGPU/kernel-args.ll
llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
llvm/test/CodeGen/AMDGPU/load-global-f32.ll
llvm/test/CodeGen/AMDGPU/load-global-i32.ll
llvm/test/CodeGen/AMDGPU/load-local-i16.ll
llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
llvm/test/CodeGen/AMDGPU/load-local.128.ll
llvm/test/CodeGen/AMDGPU/load-local.96.ll
llvm/test/CodeGen/AMDGPU/max.i16.ll
llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll
llvm/test/CodeGen/AMDGPU/packed-fp64.ll
llvm/test/CodeGen/AMDGPU/permute_i8.ll
llvm/test/CodeGen/AMDGPU/post-ra-soft-clause-dbg-info.ll
llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
llvm/test/CodeGen/AMDGPU/rem_i128.ll
llvm/test/CodeGen/AMDGPU/select.f16.ll
llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
llvm/test/CodeGen/AMDGPU/waterfall-call-target-av-register-failure.ll
llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
index 66dc17aa46e96..53a4d9d606d51 100644
--- a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
@@ -17,6 +17,7 @@
#include "GCNSubtarget.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include <deque>
using namespace llvm;
@@ -48,6 +49,7 @@ class SIPostRABundler {
bool run(MachineFunction &MF);
private:
+ const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI;
SmallSet<Register, 16> Defs;
@@ -58,6 +60,11 @@ class SIPostRABundler {
bool isBundleCandidate(const MachineInstr &MI) const;
bool isDependentLoad(const MachineInstr &MI) const;
bool canBundle(const MachineInstr &MI, const MachineInstr &NextMI) const;
+ void addDefs(MachineInstr &MI);
+ MachineBasicBlock::instr_iterator
+ reorderLoads(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator BundleStart,
+ MachineBasicBlock::instr_iterator Next);
};
} // End anonymous namespace.
@@ -137,6 +144,150 @@ bool SIPostRABundler::canBundle(const MachineInstr &MI,
!isDependentLoad(NextMI);
}
+void SIPostRABundler::addDefs(MachineInstr &MI) {
+ for (MachineOperand &Def : MI.defs())
+ if (Def.isReg())
+ Defs.insert(Def.getReg());
+}
+
+MachineBasicBlock::instr_iterator
+SIPostRABundler::reorderLoads(MachineBasicBlock &MBB,
+ MachineBasicBlock::instr_iterator BundleStart,
+ MachineBasicBlock::instr_iterator Next) {
+ // Don't reorder ALU, store or scalar clauses.
+ if (!BundleStart->mayLoad() || BundleStart->mayStore() ||
+ SIInstrInfo::isSMRD(*BundleStart) || !BundleStart->getNumExplicitDefs())
+ return BundleStart;
+
+ // Search to find the usage distance of each defined register in the clause.
+ const unsigned SearchDistance = std::max(Defs.size(), (size_t)100);
+ SmallDenseMap<Register, unsigned> UseDistance;
+ unsigned MaxDistance = 0;
+ for (MachineBasicBlock::iterator SearchI = Next;
+ SearchI != MBB.end() && MaxDistance < SearchDistance &&
+ UseDistance.size() < Defs.size();
+ ++SearchI, ++MaxDistance) {
+ // Meta instructions should not introduce waits
+ if (SearchI->isMetaInstruction())
+ continue;
+ for (Register Reg : Defs) {
+ if (UseDistance.contains(Reg))
+ continue;
+ if (SearchI->readsRegister(Reg, TRI))
+ UseDistance.insert(std::pair(Reg, MaxDistance));
+ }
+ }
+
+ if (UseDistance.empty())
+ return BundleStart;
+
+ LLVM_DEBUG(dbgs() << "Try bundle reordering\n");
+
+ // Build schedule based on use distance of register uses.
+ // Attempt to preserve exist order (NativeOrder) where possible.
+ std::deque<std::pair<MachineInstr *, unsigned>> Schedule;
+ unsigned NativeOrder = 0, LastOrder = 0;
+ bool Reordered = false;
+ for (auto II = BundleStart; II != Next; ++II, ++NativeOrder) {
+ // Bail out if we encounter anything that seems risky to reorder.
+ if (II->isKill() || II->hasOrderedMemoryRef() ||
+ (!II->getNumExplicitDefs() && !II->isMetaInstruction())) {
+ LLVM_DEBUG(dbgs() << " Abort\n");
+ return BundleStart;
+ }
+ unsigned NewOrder = MaxDistance;
+ for (const MachineOperand &Def : II->defs())
+ if (Def.isReg())
+ NewOrder = UseDistance.lookup_or(Def.getReg(), NewOrder);
+ LLVM_DEBUG(dbgs() << " Order: " << NewOrder << "," << NativeOrder
+ << ", MI: " << *II);
+ unsigned Order = (NewOrder << 16 | NativeOrder);
+ Schedule.emplace_back(&*II, Order);
+ if (II->isMetaInstruction())
+ continue;
+ Reordered |= Order < LastOrder;
+ LastOrder = Order;
+ }
+
+ // No reordering found.
+ if (!Reordered) {
+ LLVM_DEBUG(dbgs() << " No changes\n");
+ return BundleStart;
+ }
+
+ // Apply sort on new ordering.
+ std::sort(Schedule.begin(), Schedule.end(),
+ [](std::pair<MachineInstr *, unsigned> A,
+ std::pair<MachineInstr *, unsigned> B) {
+ return A.second < B.second;
+ });
+
+ // Rebuild clause order.
+ // Schedule holds ideal order for the load operations; however, each def
+ // can only be scheduled when it will no longer clobber any uses.
+ SmallVector<MachineInstr *> Clause;
+ while (!Schedule.empty()) {
+ // Try to schedule next instruction in schedule.
+ // Iterate until we find something that can be placed.
+ auto It = Schedule.begin();
+ while (It != Schedule.end()) {
+ MachineInstr *MI = It->first;
+ LLVM_DEBUG(dbgs() << "Try schedule: " << *MI);
+
+ if (MI->getNumExplicitDefs() == 0) {
+ // No defs, always schedule.
+ LLVM_DEBUG(dbgs() << " Trivially OK\n");
+ break;
+ }
+
+ // canBundle() prevents def->use and def->def chains within a bundle;
+ // however, we need to check for use->def clobbering.
+ bool DefRegHasUse = false;
+ for (auto SearchIt = std::next(It);
+ SearchIt != Schedule.end() && !DefRegHasUse; ++SearchIt)
+ DefRegHasUse = llvm::any_of(MI->defs(), [&](const MachineOperand &Def) {
+ return Def.isReg() &&
+ SearchIt->first->readsRegister(Def.getReg(), TRI);
+ });
+ if (DefRegHasUse) {
+ // A future use would be clobbered; try next instruction in the
+ // schedule.
+ LLVM_DEBUG(dbgs() << " Clobbers uses\n");
+ It++;
+ continue;
+ }
+
+ // Safe to schedule.
+ LLVM_DEBUG(dbgs() << " OK!\n");
+ break;
+ }
+
+ // Place schedule instruction into clause order.
+ assert(It != Schedule.end());
+ MachineInstr *MI = It->first;
+ Schedule.erase(It);
+ Clause.push_back(MI);
+
+ // Clear kill flags for later uses.
+ for (auto &Use : MI->all_uses()) {
+ if (!Use.isReg() || !Use.isKill())
+ continue;
+ Register UseReg = Use.getReg();
+ if (llvm::any_of(Schedule, [&](std::pair<MachineInstr *, unsigned> &SI) {
+ return SI.first->readsRegister(UseReg, TRI);
+ }))
+ Use.setIsKill(false);
+ }
+ }
+
+ // Apply order to instructions.
+ for (MachineInstr *MI : Clause)
+ MI->moveBefore(&*Next);
+
+ // Return new start of bundle.
+ return Clause[0]->getIterator();
+}
+
bool SIPostRABundlerLegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -151,6 +302,8 @@ PreservedAnalyses SIPostRABundlerPass::run(MachineFunction &MF,
bool SIPostRABundler::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ TII = ST.getInstrInfo();
TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
BitVector BundleUsedRegUnits(TRI->getNumRegUnits());
BitVector KillUsedRegUnits(TRI->getNumRegUnits());
@@ -176,9 +329,7 @@ bool SIPostRABundler::run(MachineFunction &MF) {
continue;
assert(Defs.empty());
-
- if (I->getNumExplicitDefs() != 0)
- Defs.insert(I->defs().begin()->getReg());
+ addDefs(*I);
MachineBasicBlock::instr_iterator BundleStart = I;
MachineBasicBlock::instr_iterator BundleEnd = I;
@@ -189,8 +340,7 @@ bool SIPostRABundler::run(MachineFunction &MF) {
assert(BundleEnd != I);
if (canBundle(*BundleEnd, *I)) {
BundleEnd = I;
- if (I->getNumExplicitDefs() != 0)
- Defs.insert(I->defs().begin()->getReg());
+ addDefs(*I);
++ClauseLength;
} else if (!I->isMetaInstruction() ||
I->getOpcode() == AMDGPU::SCHED_BARRIER) {
@@ -242,6 +392,7 @@ bool SIPostRABundler::run(MachineFunction &MF) {
BundleUsedRegUnits.reset();
}
+ BundleStart = reorderLoads(MBB, BundleStart, Next);
finalizeBundle(MBB, BundleStart, Next);
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index d3eed3e7b4fd9..5487024131d98 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -558,14 +558,14 @@ define void @add_v9i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v9i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:16
+; GFX9-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off
; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_or_b32_sdwa v14, v0, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v6, v10
@@ -738,9 +738,9 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:18
; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:20
-; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:18
; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:20
+; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:16
; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:16
@@ -749,9 +749,9 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_or_b32_sdwa v17, v0, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_pk_add_u16 v0, v10, v6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
index ce4de6e628573..ee803fa778e54 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
@@ -790,20 +790,20 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX10-CONTRACT: ; %bb.0: ; %.entry
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-CONTRACT-NEXT: s_clause 0x8
-; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX10-CONTRACT-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GFX10-CONTRACT-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; GFX10-CONTRACT-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; GFX10-CONTRACT-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
; GFX10-CONTRACT-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-CONTRACT-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
; GFX10-CONTRACT-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(7)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(5)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(3)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -817,20 +817,20 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX10-DENORM: ; %bb.0: ; %.entry
; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DENORM-NEXT: s_clause 0x8
-; GFX10-DENORM-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-DENORM-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX10-DENORM-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GFX10-DENORM-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; GFX10-DENORM-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; GFX10-DENORM-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
; GFX10-DENORM-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-DENORM-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-DENORM-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
; GFX10-DENORM-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(6)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(7)
; GFX10-DENORM-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(4)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(5)
; GFX10-DENORM-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(3)
; GFX10-DENORM-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -844,20 +844,20 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX11-CONTRACT: ; %bb.0: ; %.entry
; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-CONTRACT-NEXT: s_clause 0x8
-; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
; GFX11-CONTRACT-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-CONTRACT-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-CONTRACT-NEXT: scratch_load_b32 v34, off, s32 offset:12
; GFX11-CONTRACT-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX11-CONTRACT-NEXT: scratch_load_b32 v36, off, s32 offset:20
; GFX11-CONTRACT-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
; GFX11-CONTRACT-NEXT: scratch_load_b32 v38, off, s32 offset:28
; GFX11-CONTRACT-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(7)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(5)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(3)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -873,20 +873,20 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX11-DENORM: ; %bb.0: ; %.entry
; GFX11-DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-DENORM-NEXT: s_clause 0x8
-; GFX11-DENORM-NEXT: scratch_load_b32 v31, off, s32
; GFX11-DENORM-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-DENORM-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-DENORM-NEXT: scratch_load_b32 v34, off, s32 offset:12
; GFX11-DENORM-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX11-DENORM-NEXT: scratch_load_b32 v36, off, s32 offset:20
; GFX11-DENORM-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-DENORM-NEXT: scratch_load_b32 v31, off, s32
; GFX11-DENORM-NEXT: scratch_load_b32 v38, off, s32 offset:28
; GFX11-DENORM-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(6)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(7)
; GFX11-DENORM-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(4)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(5)
; GFX11-DENORM-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(2)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(3)
; GFX11-DENORM-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX11-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -961,20 +961,20 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX10-CONTRACT: ; %bb.0: ; %.entry
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-CONTRACT-NEXT: s_clause 0x8
-; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX10-CONTRACT-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GFX10-CONTRACT-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; GFX10-CONTRACT-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; GFX10-CONTRACT-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
; GFX10-CONTRACT-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-CONTRACT-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
; GFX10-CONTRACT-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(7)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(5)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(3)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -988,20 +988,20 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX10-DENORM: ; %bb.0: ; %.entry
; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DENORM-NEXT: s_clause 0x8
-; GFX10-DENORM-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-DENORM-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX10-DENORM-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GFX10-DENORM-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; GFX10-DENORM-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; GFX10-DENORM-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
; GFX10-DENORM-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-DENORM-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-DENORM-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
; GFX10-DENORM-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(6)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(7)
; GFX10-DENORM-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(4)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(5)
; GFX10-DENORM-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-DENORM-NEXT: s_waitcnt vmcnt(2)
+; GFX10-DENORM-NEXT: s_waitcnt vmcnt(3)
; GFX10-DENORM-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX10-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -1015,20 +1015,20 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX11-CONTRACT: ; %bb.0: ; %.entry
; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-CONTRACT-NEXT: s_clause 0x8
-; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
; GFX11-CONTRACT-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-CONTRACT-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-CONTRACT-NEXT: scratch_load_b32 v34, off, s32 offset:12
; GFX11-CONTRACT-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX11-CONTRACT-NEXT: scratch_load_b32 v36, off, s32 offset:20
; GFX11-CONTRACT-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
; GFX11-CONTRACT-NEXT: scratch_load_b32 v38, off, s32 offset:28
; GFX11-CONTRACT-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(7)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(5)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(3)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX11-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
@@ -1044,20 +1044,20 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX11-DENORM: ; %bb.0: ; %.entry
; GFX11-DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-DENORM-NEXT: s_clause 0x8
-; GFX11-DENORM-NEXT: scratch_load_b32 v31, off, s32
; GFX11-DENORM-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-DENORM-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-DENORM-NEXT: scratch_load_b32 v34, off, s32 offset:12
; GFX11-DENORM-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX11-DENORM-NEXT: scratch_load_b32 v36, off, s32 offset:20
; GFX11-DENORM-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-DENORM-NEXT: scratch_load_b32 v31, off, s32
; GFX11-DENORM-NEXT: scratch_load_b32 v38, off, s32 offset:28
; GFX11-DENORM-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(6)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(7)
; GFX11-DENORM-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(4)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(5)
; GFX11-DENORM-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-DENORM-NEXT: s_waitcnt vmcnt(2)
+; GFX11-DENORM-NEXT: s_waitcnt vmcnt(3)
; GFX11-DENORM-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
; GFX11-DENORM-NEXT: s_waitcnt vmcnt(0)
; GFX11-DENORM-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
index 33d7bd4c75ace..df4b6b85f1608 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
@@ -69,6 +69,14 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: s_or_saveexec_b32 s21, -1
; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b32 exec_lo, s21
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
@@ -77,22 +85,21 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_waitcnt vmcnt(15)
; CHECK-NEXT: v_readfirstlane_b32 s12, v7
+; CHECK-NEXT: s_waitcnt vmcnt(14)
; CHECK-NEXT: v_readfirstlane_b32 s10, v6
+; CHECK-NEXT: s_waitcnt vmcnt(13)
; CHECK-NEXT: v_readfirstlane_b32 s9, v5
+; CHECK-NEXT: s_waitcnt vmcnt(12)
; CHECK-NEXT: v_readfirstlane_b32 s8, v4
+; CHECK-NEXT: s_waitcnt vmcnt(11)
; CHECK-NEXT: v_readfirstlane_b32 s7, v3
+; CHECK-NEXT: s_waitcnt vmcnt(10)
; CHECK-NEXT: v_readfirstlane_b32 s6, v2
+; CHECK-NEXT: s_waitcnt vmcnt(9)
; CHECK-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: v_readfirstlane_b32 s4, v0
; CHECK-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
; CHECK-NEXT: s_mov_b32 s13, s10
@@ -110,12 +117,16 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: v_writelane_b32 v16, s17, 10
; CHECK-NEXT: v_writelane_b32 v16, s18, 11
; CHECK-NEXT: v_writelane_b32 v16, s19, 12
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: v_mov_b32_e32 v6, v8
; CHECK-NEXT: v_mov_b32_e32 v7, v9
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: v_mov_b32_e32 v4, v10
; CHECK-NEXT: v_mov_b32_e32 v5, v11
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: v_mov_b32_e32 v2, v12
; CHECK-NEXT: v_mov_b32_e32 v3, v13
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v0, v14
; CHECK-NEXT: v_mov_b32_e32 v1, v15
; CHECK-NEXT: s_mov_b64 s[4:5], s[12:13]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
index 84bdcdeb1ec46..b5eed1212c914 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.ll
@@ -99,10 +99,10 @@ define amdgpu_cs void @memcpy_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src)
; LOOP-NEXT: v_or_b32_e32 v5, v5, v7
; LOOP-NEXT: buffer_load_ubyte v7, v[2:3], s[0:3], 0 addr64 offset:25
; LOOP-NEXT: v_or_b32_e32 v17, v17, v19
-; LOOP-NEXT: buffer_load_ubyte v19, v[2:3], s[0:3], 0 addr64 offset:28
; LOOP-NEXT: buffer_load_ubyte v20, v[2:3], s[0:3], 0 addr64 offset:29
-; LOOP-NEXT: buffer_load_ubyte v21, v[2:3], s[0:3], 0 addr64 offset:30
+; LOOP-NEXT: buffer_load_ubyte v19, v[2:3], s[0:3], 0 addr64 offset:28
; LOOP-NEXT: buffer_load_ubyte v22, v[2:3], s[0:3], 0 addr64 offset:31
+; LOOP-NEXT: buffer_load_ubyte v21, v[2:3], s[0:3], 0 addr64 offset:30
; LOOP-NEXT: v_or_b32_e32 v5, v9, v5
; LOOP-NEXT: v_or_b32_e32 v9, v17, v13
; LOOP-NEXT: v_bfe_u32 v13, v4, 8, 8
@@ -138,8 +138,9 @@ define amdgpu_cs void @memcpy_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src)
; LOOP-NEXT: v_lshlrev_b32_e32 v15, 8, v20
; LOOP-NEXT: v_lshlrev_b32_e32 v7, 8, v7
; LOOP-NEXT: v_or_b32_e32 v15, v15, v19
-; LOOP-NEXT: s_waitcnt vmcnt(12)
+; LOOP-NEXT: s_waitcnt vmcnt(13)
; LOOP-NEXT: v_lshlrev_b32_e32 v19, 24, v22
+; LOOP-NEXT: s_waitcnt vmcnt(12)
; LOOP-NEXT: v_lshlrev_b32_e32 v20, 16, v21
; LOOP-NEXT: v_or_b32_e32 v7, v7, v23
; LOOP-NEXT: v_or_b32_e32 v19, v19, v20
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 1ea44e2c69d83..c2c60f2220fee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -32,12 +32,12 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX12-NOUNALIGNED-NEXT: s_clause 0xb
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v2, v[0:1], off
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v3, v[0:1], off offset:1
-; GFX12-NOUNALIGNED-NEXT: global_load_u8 v4, v[0:1], off offset:2
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v5, v[0:1], off offset:3
+; GFX12-NOUNALIGNED-NEXT: global_load_u8 v4, v[0:1], off offset:2
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v6, v[0:1], off offset:4
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v7, v[0:1], off offset:5
-; GFX12-NOUNALIGNED-NEXT: global_load_u8 v8, v[0:1], off offset:6
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v9, v[0:1], off offset:7
+; GFX12-NOUNALIGNED-NEXT: global_load_u8 v8, v[0:1], off offset:6
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v10, v[0:1], off offset:8
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v11, v[0:1], off offset:9
; GFX12-NOUNALIGNED-NEXT: global_load_u8 v12, v[0:1], off offset:11
@@ -45,15 +45,15 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0xa
; GFX12-NOUNALIGNED-NEXT: v_lshl_or_b32 v1, v3, 8, v2
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x9
-; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x8
; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 24, v5
+; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x8
+; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x6
; GFX12-NOUNALIGNED-NEXT: v_lshl_or_b32 v4, v7, 8, v6
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x5
-; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v6, 16, v8
-; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x4
; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v5, 24, v9
+; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x4
+; GFX12-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v6, 16, v8
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x2
; GFX12-NOUNALIGNED-NEXT: v_lshl_or_b32 v7, v11, 8, v10
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x1
@@ -81,12 +81,12 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX1250-NOUNALIGNED-NEXT: s_clause 0xb
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v2, v[0:1], off nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v3, v[0:1], off offset:1 nv
-; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v4, v[0:1], off offset:2 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v5, v[0:1], off offset:3 nv
+; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v4, v[0:1], off offset:2 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v6, v[0:1], off offset:4 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v7, v[0:1], off offset:5 nv
-; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v8, v[0:1], off offset:6 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v9, v[0:1], off offset:7 nv
+; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v8, v[0:1], off offset:6 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v10, v[0:1], off offset:8 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v11, v[0:1], off offset:9 nv
; GFX1250-NOUNALIGNED-NEXT: global_load_u8 v12, v[0:1], off offset:11 nv
@@ -95,11 +95,11 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX1250-NOUNALIGNED-NEXT: s_wait_xcnt 0x0
; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v0, v3, 8, v2
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x8
-; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v2, 16, v4 :: v_dual_lshlrev_b32 v1, 24, v5
+; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v1, 24, v5 :: v_dual_lshlrev_b32 v2, 16, v4
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x6
; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x4
-; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v5, 16, v8 :: v_dual_lshlrev_b32 v4, 24, v9
+; GFX1250-NOUNALIGNED-NEXT: v_dual_lshlrev_b32 v4, 24, v9 :: v_dual_lshlrev_b32 v5, 16, v8
; GFX1250-NOUNALIGNED-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX1250-NOUNALIGNED-NEXT: s_wait_loadcnt 0x2
; GFX1250-NOUNALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
@@ -122,12 +122,12 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v2, v[0:1], off
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v3, v[0:1], off offset:1
-; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v4, v[0:1], off offset:2
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v5, v[0:1], off offset:3
+; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v4, v[0:1], off offset:2
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v6, v[0:1], off offset:4
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v7, v[0:1], off offset:5
-; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v8, v[0:1], off offset:6
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v9, v[0:1], off offset:7
+; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v8, v[0:1], off offset:6
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v10, v[0:1], off offset:8
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v11, v[0:1], off offset:9
; GFX9-NOUNALIGNED-NEXT: global_load_ubyte v12, v[0:1], off offset:11
@@ -135,16 +135,16 @@ define <3 x i32> @v_load_constant_v3i32_align1(ptr addrspace(4) %ptr) {
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(10)
; GFX9-NOUNALIGNED-NEXT: v_lshl_or_b32 v0, v3, 8, v2
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(8)
; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v1, 24, v5
+; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX9-NOUNALIGNED-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(6)
; GFX9-NOUNALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v5, 16, v8
-; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(4)
; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v4, 24, v9
+; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NOUNALIGNED-NEXT: v_lshlrev_b32_e32 v5, 16, v8
; GFX9-NOUNALIGNED-NEXT: v_or3_b32 v1, v4, v5, v3
; GFX9-NOUNALIGNED-NEXT: s_waitcnt vmcnt(2)
; GFX9-NOUNALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
index 29f7d670cbc7b..b8b4e0aa9c802 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
@@ -46,36 +46,39 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u8 v1, v0
; GFX9-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
; GFX9-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
; GFX9-NEXT: ds_read_u8 v5, v0 offset:4
; GFX9-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
; GFX9-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
; GFX9-NEXT: s_waitcnt lgkmcnt(6)
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 8, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NEXT: s_waitcnt lgkmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or3_b32 v4, v2, v3, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-NEXT: v_lshl_or_b32 v1, v6, 8, v5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v7
; GFX9-NEXT: v_or3_b32 v1, v2, v3, v1
; GFX9-NEXT: ds_read_u8 v2, v0 offset:8
; GFX9-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX9-NEXT: ds_read_u8 v5, v0 offset:10
; GFX9-NEXT: ds_read_u8 v6, v0 offset:11
+; GFX9-NEXT: ds_read_u8 v5, v0 offset:10
; GFX9-NEXT: ds_read_u8 v7, v0 offset:12
; GFX9-NEXT: ds_read_u8 v8, v0 offset:13
; GFX9-NEXT: ds_read_u8 v9, v0 offset:14
; GFX9-NEXT: ds_read_u8 v0, v0 offset:15
; GFX9-NEXT: s_waitcnt lgkmcnt(6)
; GFX9-NEXT: v_lshl_or_b32 v2, v3, 8, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NEXT: s_waitcnt lgkmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or3_b32 v2, v3, v5, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
@@ -91,47 +94,52 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v4, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v7
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
+; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
; GFX7-NEXT: ds_read_u8 v8, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
; GFX7-NEXT: ds_read_u8 v0, v0 offset:15
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
@@ -147,16 +155,16 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_u8 v1, v0
; GFX10-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX10-NEXT: ds_read_u8 v3, v0 offset:2
; GFX10-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX10-NEXT: ds_read_u8 v3, v0 offset:2
; GFX10-NEXT: ds_read_u8 v5, v0 offset:4
; GFX10-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX10-NEXT: ds_read_u8 v7, v0 offset:6
; GFX10-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX10-NEXT: ds_read_u8 v7, v0 offset:6
; GFX10-NEXT: ds_read_u8 v9, v0 offset:8
; GFX10-NEXT: ds_read_u8 v10, v0 offset:9
-; GFX10-NEXT: ds_read_u8 v11, v0 offset:10
; GFX10-NEXT: ds_read_u8 v12, v0 offset:11
+; GFX10-NEXT: ds_read_u8 v11, v0 offset:10
; GFX10-NEXT: ds_read_u8 v13, v0 offset:12
; GFX10-NEXT: ds_read_u8 v14, v0 offset:13
; GFX10-NEXT: ds_read_u8 v15, v0 offset:15
@@ -164,21 +172,21 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX10-NEXT: s_waitcnt lgkmcnt(14)
; GFX10-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(13)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: s_waitcnt lgkmcnt(12)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX10-NEXT: s_waitcnt lgkmcnt(12)
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(10)
; GFX10-NEXT: v_lshl_or_b32 v4, v6, 8, v5
; GFX10-NEXT: s_waitcnt lgkmcnt(9)
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX10-NEXT: s_waitcnt lgkmcnt(8)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 24, v8
+; GFX10-NEXT: s_waitcnt lgkmcnt(8)
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; GFX10-NEXT: s_waitcnt lgkmcnt(6)
; GFX10-NEXT: v_lshl_or_b32 v7, v10, 8, v9
; GFX10-NEXT: s_waitcnt lgkmcnt(5)
-; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v11
-; GFX10-NEXT: s_waitcnt lgkmcnt(4)
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 24, v12
+; GFX10-NEXT: s_waitcnt lgkmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX10-NEXT: s_waitcnt lgkmcnt(2)
; GFX10-NEXT: v_lshl_or_b32 v10, v14, 8, v13
; GFX10-NEXT: s_waitcnt lgkmcnt(1)
@@ -196,16 +204,16 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_u8 v1, v0
; GFX11-NEXT: ds_load_u8 v2, v0 offset:1
-; GFX11-NEXT: ds_load_u8 v3, v0 offset:2
; GFX11-NEXT: ds_load_u8 v4, v0 offset:3
+; GFX11-NEXT: ds_load_u8 v3, v0 offset:2
; GFX11-NEXT: ds_load_u8 v5, v0 offset:4
; GFX11-NEXT: ds_load_u8 v6, v0 offset:5
-; GFX11-NEXT: ds_load_u8 v7, v0 offset:6
; GFX11-NEXT: ds_load_u8 v8, v0 offset:7
+; GFX11-NEXT: ds_load_u8 v7, v0 offset:6
; GFX11-NEXT: ds_load_u8 v9, v0 offset:8
; GFX11-NEXT: ds_load_u8 v10, v0 offset:9
-; GFX11-NEXT: ds_load_u8 v11, v0 offset:10
; GFX11-NEXT: ds_load_u8 v12, v0 offset:11
+; GFX11-NEXT: ds_load_u8 v11, v0 offset:10
; GFX11-NEXT: ds_load_u8 v13, v0 offset:12
; GFX11-NEXT: ds_load_u8 v14, v0 offset:13
; GFX11-NEXT: ds_load_u8 v15, v0 offset:15
@@ -213,21 +221,21 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt lgkmcnt(14)
; GFX11-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(13)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_waitcnt lgkmcnt(12)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX11-NEXT: s_waitcnt lgkmcnt(12)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(10)
; GFX11-NEXT: v_lshl_or_b32 v4, v6, 8, v5
; GFX11-NEXT: s_waitcnt lgkmcnt(9)
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_waitcnt lgkmcnt(8)
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 24, v8
+; GFX11-NEXT: s_waitcnt lgkmcnt(8)
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; GFX11-NEXT: s_waitcnt lgkmcnt(6)
; GFX11-NEXT: v_lshl_or_b32 v7, v10, 8, v9
; GFX11-NEXT: s_waitcnt lgkmcnt(5)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
-; GFX11-NEXT: s_waitcnt lgkmcnt(4)
; GFX11-NEXT: v_lshlrev_b32_e32 v8, 24, v12
+; GFX11-NEXT: s_waitcnt lgkmcnt(4)
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX11-NEXT: s_waitcnt lgkmcnt(2)
; GFX11-NEXT: v_lshl_or_b32 v10, v14, 8, v13
; GFX11-NEXT: s_waitcnt lgkmcnt(1)
@@ -270,25 +278,28 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u16 v1, v0
; GFX7-NEXT: ds_read_u16 v2, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v1, v0
; GFX7-NEXT: ds_read_u16 v4, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
-; GFX7-NEXT: ds_read_u16 v7, v0 offset:12
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
; GFX7-NEXT: ds_read_u16 v8, v0 offset:14
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: ds_read_u16 v7, v0 offset:12
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
index e644381b64e03..e2a5bb328d09a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
@@ -46,16 +46,17 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u8 v1, v0
; GFX9-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
; GFX9-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
; GFX9-NEXT: ds_read_u8 v5, v0 offset:4
; GFX9-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
; GFX9-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
; GFX9-NEXT: s_waitcnt lgkmcnt(6)
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 8, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NEXT: s_waitcnt lgkmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or3_b32 v3, v2, v3, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
@@ -64,8 +65,9 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX9-NEXT: ds_read_u8 v4, v0 offset:9
; GFX9-NEXT: ds_read_u8 v5, v0 offset:10
; GFX9-NEXT: ds_read_u8 v0, v0 offset:11
-; GFX9-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NEXT: s_waitcnt lgkmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v6, 24, v8
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-NEXT: v_lshl_or_b32 v2, v4, 8, v2
@@ -82,36 +84,38 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:9
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
; GFX7-NEXT: ds_read_u8 v0, v0 offset:11
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
@@ -127,12 +131,12 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_u8 v1, v0
; GFX10-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX10-NEXT: ds_read_u8 v3, v0 offset:2
; GFX10-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX10-NEXT: ds_read_u8 v3, v0 offset:2
; GFX10-NEXT: ds_read_u8 v5, v0 offset:4
; GFX10-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX10-NEXT: ds_read_u8 v7, v0 offset:6
; GFX10-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX10-NEXT: ds_read_u8 v7, v0 offset:6
; GFX10-NEXT: ds_read_u8 v9, v0 offset:8
; GFX10-NEXT: ds_read_u8 v10, v0 offset:9
; GFX10-NEXT: ds_read_u8 v11, v0 offset:11
@@ -140,15 +144,15 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX10-NEXT: s_waitcnt lgkmcnt(10)
; GFX10-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(9)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: s_waitcnt lgkmcnt(8)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX10-NEXT: s_waitcnt lgkmcnt(8)
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(6)
; GFX10-NEXT: v_lshl_or_b32 v4, v6, 8, v5
; GFX10-NEXT: s_waitcnt lgkmcnt(5)
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX10-NEXT: s_waitcnt lgkmcnt(4)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 24, v8
+; GFX10-NEXT: s_waitcnt lgkmcnt(4)
+; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; GFX10-NEXT: s_waitcnt lgkmcnt(2)
; GFX10-NEXT: v_lshl_or_b32 v7, v10, 8, v9
; GFX10-NEXT: s_waitcnt lgkmcnt(1)
@@ -165,12 +169,12 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_u8 v1, v0
; GFX11-NEXT: ds_load_u8 v2, v0 offset:1
-; GFX11-NEXT: ds_load_u8 v3, v0 offset:2
; GFX11-NEXT: ds_load_u8 v4, v0 offset:3
+; GFX11-NEXT: ds_load_u8 v3, v0 offset:2
; GFX11-NEXT: ds_load_u8 v5, v0 offset:4
; GFX11-NEXT: ds_load_u8 v6, v0 offset:5
-; GFX11-NEXT: ds_load_u8 v7, v0 offset:6
; GFX11-NEXT: ds_load_u8 v8, v0 offset:7
+; GFX11-NEXT: ds_load_u8 v7, v0 offset:6
; GFX11-NEXT: ds_load_u8 v9, v0 offset:8
; GFX11-NEXT: ds_load_u8 v10, v0 offset:9
; GFX11-NEXT: ds_load_u8 v11, v0 offset:11
@@ -178,15 +182,15 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX11-NEXT: s_waitcnt lgkmcnt(10)
; GFX11-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(9)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_waitcnt lgkmcnt(8)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX11-NEXT: s_waitcnt lgkmcnt(8)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(6)
; GFX11-NEXT: v_lshl_or_b32 v4, v6, 8, v5
; GFX11-NEXT: s_waitcnt lgkmcnt(5)
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_waitcnt lgkmcnt(4)
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 24, v8
+; GFX11-NEXT: s_waitcnt lgkmcnt(4)
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
; GFX11-NEXT: s_waitcnt lgkmcnt(2)
; GFX11-NEXT: v_lshl_or_b32 v7, v10, 8, v9
; GFX11-NEXT: s_waitcnt lgkmcnt(1)
@@ -224,20 +228,22 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u16 v1, v0
; GFX7-NEXT: ds_read_u16 v2, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v1, v0
; GFX7-NEXT: ds_read_u16 v4, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
index 7d3237e792c24..6c93829fbf9f2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
@@ -22,47 +22,52 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v4, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v7
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
+; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
; GFX7-NEXT: ds_read_u8 v8, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
; GFX7-NEXT: ds_read_u8 v0, v0 offset:15
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
@@ -104,36 +109,38 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v1, v0
; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:9
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
; GFX7-NEXT: ds_read_u8 v0, v0 offset:11
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 2b1650e429b86..37df670987822 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -4084,8 +4084,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; kill: killed $vgpr36
@@ -4247,7 +4247,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; kill: killed $vgpr36
; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -4485,6 +4485,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB12_4
; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_i32_e32 v31, vcc, 3, v31
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_i32_e32 v32, vcc, 3, v32
@@ -5265,8 +5266,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: ; implicit-def: $vgpr39
; VI-NEXT: ; kill: killed $vgpr39
@@ -5865,11 +5866,11 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v14, v14, v35, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v16, v38, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v35, v35, v48, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -6144,8 +6145,8 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr39
; GFX9-NEXT: ; kill: killed $vgpr39
@@ -6331,13 +6332,14 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB12_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v32
@@ -6527,6 +6529,7 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB12_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_add_u32_e32 v32, 3, v32
; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: v_add_u32_e32 v31, 3, v31
@@ -6783,11 +6786,11 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -11941,12 +11944,12 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v52
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:28
@@ -11962,20 +11965,21 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:60
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:52
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204
@@ -11985,28 +11989,29 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236
@@ -12016,28 +12021,29 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:268
@@ -12047,28 +12053,29 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:312
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:300
@@ -12078,28 +12085,29 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:344
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
@@ -12109,36 +12117,38 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:376
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:364
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
@@ -13354,9 +13364,9 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:112
@@ -13388,18 +13398,17 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:4
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:288
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -14633,9 +14642,9 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
@@ -14667,10 +14676,10 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_waitcnt vmcnt(31)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -14744,13 +14753,13 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -15935,6 +15944,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
@@ -15966,8 +15976,8 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
@@ -15999,8 +16009,8 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
@@ -16029,7 +16039,6 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
@@ -16067,7 +16076,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -16781,6 +16790,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v86, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:376
@@ -16812,8 +16822,8 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v147, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:268
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:264
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:248
@@ -16845,8 +16855,8 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v59, off, s32 offset:144
; GFX11-FAKE16-NEXT: scratch_load_u16 v60, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:136
-; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v74, off, s32 offset:120
@@ -16875,7 +16885,6 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v120, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v121, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v122, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v123, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v125, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v126, off, s32 offset:8
@@ -16897,7 +16906,7 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v6 :: v_dual_mov_b32 v83, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, v5 :: v_dual_mov_b32 v81, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v2 :: v_dual_mov_b32 v84, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -17744,19 +17753,19 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(9) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:268
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:244
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
@@ -17771,52 +17780,49 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v38, 8, v36
; SI-NEXT: v_lshlrev_b32_e32 v36, 8, v34
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v32
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v49
; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v39
-; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
-; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
-; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
+; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v31
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60
@@ -17828,6 +17834,9 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
@@ -17861,8 +17870,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
@@ -18761,10 +18770,10 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v14
; VI-NEXT: v_readfirstlane_b32 s47, v13
; VI-NEXT: v_readfirstlane_b32 s76, v0
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: v_readfirstlane_b32 s6, v30
; VI-NEXT: v_readfirstlane_b32 s7, v29
@@ -18793,7 +18802,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s73, v3
; VI-NEXT: v_readfirstlane_b32 s74, v2
; VI-NEXT: v_readfirstlane_b32 s75, v1
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -18898,23 +18907,16 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -18935,6 +18937,13 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -18967,7 +18976,6 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
@@ -18975,6 +18983,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
@@ -19648,10 +19657,10 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v14
; GFX9-NEXT: v_readfirstlane_b32 s47, v13
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; GFX9-NEXT: v_readfirstlane_b32 s6, v30
; GFX9-NEXT: v_readfirstlane_b32 s7, v29
@@ -19680,7 +19689,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s73, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
; GFX9-NEXT: v_readfirstlane_b32 s75, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -19785,23 +19794,16 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -19822,6 +19824,13 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -19845,68 +19854,68 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
@@ -20005,6 +20014,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_or_b32_e32 v16, v17, v16
; GFX9-NEXT: v_perm_b32 v17, v59, v18, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX9-NEXT: s_waitcnt vmcnt(45)
; GFX9-NEXT: v_perm_b32 v18, v41, v40, s4
; GFX9-NEXT: v_or_b32_e32 v17, v18, v17
; GFX9-NEXT: v_perm_b32 v18, v57, v19, s4
@@ -20118,6 +20128,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB15_3
; GFX9-NEXT: .LBB15_2:
+; GFX9-NEXT: s_waitcnt vmcnt(48)
; GFX9-NEXT: v_mov_b32_e32 v57, v56
; GFX9-NEXT: v_mov_b32_e32 v47, v46
; GFX9-NEXT: v_mov_b32_e32 v45, v26
@@ -20530,6 +20541,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:304
@@ -20561,8 +20573,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:176
@@ -20594,8 +20606,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:48
@@ -20604,7 +20616,6 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
@@ -20644,7 +20655,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB15_2
@@ -20895,6 +20906,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -21109,6 +21121,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: ; meta instruction
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:304
@@ -21140,8 +21153,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:176
@@ -21173,8 +21186,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v163, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:48
@@ -21183,7 +21196,6 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
@@ -21223,7 +21235,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB15_2
@@ -21474,6 +21486,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -26547,8 +26560,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
@@ -26556,7 +26569,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
; SI-NEXT: v_mov_b32_e32 v9, v25
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
@@ -26573,8 +26586,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
@@ -26584,8 +26597,6 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
; SI-NEXT: v_mov_b32_e32 v12, v61
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
@@ -26608,15 +26619,14 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
; SI-NEXT: v_mov_b32_e32 v15, v60
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
@@ -26648,6 +26658,7 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
; SI-NEXT: v_mov_b32_e32 v39, v62
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -26741,9 +26752,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB19_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -39189,8 +39200,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; kill: killed $vgpr36
@@ -39352,7 +39363,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; kill: killed $vgpr36
; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -39590,6 +39601,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: s_cbranch_execz .LBB36_4
; SI-NEXT: ; %bb.3: ; %cmp.true
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_add_f32_e32 v31, 1.0, v31
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_add_f32_e32 v32, 1.0, v32
@@ -40370,8 +40382,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: ; implicit-def: $vgpr39
; VI-NEXT: ; kill: killed $vgpr39
@@ -40970,11 +40982,11 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v14, v14, v35, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v16, v38, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v35, v35, v48, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -41249,8 +41261,8 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr39
; GFX9-NEXT: ; kill: killed $vgpr39
@@ -41436,13 +41448,14 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB36_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v32
@@ -41632,6 +41645,7 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB36_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_add_f32_e32 v32, 1.0, v32
; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: v_add_f32_e32 v31, 1.0, v31
@@ -41888,11 +41902,11 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -45389,11 +45403,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_perm_b32 v24, v24, v15, s4
; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: v_perm_b32 v48, v44, v48, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v49, v15, v49, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -45402,9 +45416,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v22, v22, v15, s4
; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v15, v15, v50, s4
; VI-NEXT: v_lshlrev_b32_e32 v50, 16, v55
; VI-NEXT: v_or_b32_e32 v20, v20, v50
@@ -46590,10 +46604,10 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v39, v44, v49, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v38, v47, v38, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v49, v15, v49, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -46602,9 +46616,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v22, v22, v15, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v15, v15, v50, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
; GFX9-NEXT: v_or_b32_e32 v20, v20, v50
@@ -47951,12 +47965,12 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v52
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:28
@@ -47972,20 +47986,21 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:60
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:52
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204
@@ -47995,28 +48010,29 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236
@@ -48026,28 +48042,29 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:268
@@ -48057,28 +48074,29 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:312
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:300
@@ -48088,28 +48106,29 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:344
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
@@ -48119,36 +48138,38 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:376
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:364
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
@@ -49364,9 +49385,9 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:112
@@ -49398,18 +49419,17 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:4
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:288
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -50643,9 +50663,9 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
@@ -50677,10 +50697,10 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_waitcnt vmcnt(31)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -50754,13 +50774,13 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -51945,6 +51965,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
@@ -51976,8 +51997,8 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
@@ -52009,8 +52030,8 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
@@ -52039,7 +52060,6 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
@@ -52077,7 +52097,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -52791,6 +52811,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v86, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:376
@@ -52822,8 +52843,8 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v147, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:268
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:264
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:248
@@ -52855,8 +52876,8 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v59, off, s32 offset:144
; GFX11-FAKE16-NEXT: scratch_load_u16 v60, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:136
-; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v74, off, s32 offset:120
@@ -52885,7 +52906,6 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v120, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v121, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v122, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v123, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v125, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v126, off, s32 offset:8
@@ -52907,7 +52927,7 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v6 :: v_dual_mov_b32 v83, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, v5 :: v_dual_mov_b32 v81, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v2 :: v_dual_mov_b32 v84, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -53754,19 +53774,19 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(9) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:268
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:244
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
@@ -53781,52 +53801,49 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v38, 8, v36
; SI-NEXT: v_lshlrev_b32_e32 v36, 8, v34
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v32
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v49
; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v39
-; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
-; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
-; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
+; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v31
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60
@@ -53838,6 +53855,9 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
@@ -53871,8 +53891,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
@@ -54771,10 +54791,10 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v14
; VI-NEXT: v_readfirstlane_b32 s47, v13
; VI-NEXT: v_readfirstlane_b32 s76, v0
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: v_readfirstlane_b32 s6, v30
; VI-NEXT: v_readfirstlane_b32 s7, v29
@@ -54803,7 +54823,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s73, v3
; VI-NEXT: v_readfirstlane_b32 s74, v2
; VI-NEXT: v_readfirstlane_b32 s75, v1
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -54908,23 +54928,16 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -54945,6 +54958,13 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -54977,7 +54997,6 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
@@ -54985,6 +55004,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
@@ -55658,10 +55678,10 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v14
; GFX9-NEXT: v_readfirstlane_b32 s47, v13
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; GFX9-NEXT: v_readfirstlane_b32 s6, v30
; GFX9-NEXT: v_readfirstlane_b32 s7, v29
@@ -55690,7 +55710,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s73, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
; GFX9-NEXT: v_readfirstlane_b32 s75, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -55795,23 +55815,16 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -55832,6 +55845,13 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -55855,68 +55875,68 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB39_2
@@ -56015,6 +56035,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v16, v17, v16
; GFX9-NEXT: v_perm_b32 v17, v59, v18, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX9-NEXT: s_waitcnt vmcnt(45)
; GFX9-NEXT: v_perm_b32 v18, v41, v40, s4
; GFX9-NEXT: v_or_b32_e32 v17, v18, v17
; GFX9-NEXT: v_perm_b32 v18, v57, v19, s4
@@ -56128,6 +56149,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB39_3
; GFX9-NEXT: .LBB39_2:
+; GFX9-NEXT: s_waitcnt vmcnt(48)
; GFX9-NEXT: v_mov_b32_e32 v57, v56
; GFX9-NEXT: v_mov_b32_e32 v47, v46
; GFX9-NEXT: v_mov_b32_e32 v45, v26
@@ -56540,6 +56562,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:304
@@ -56571,8 +56594,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:176
@@ -56604,8 +56627,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:48
@@ -56614,7 +56637,6 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
@@ -56654,7 +56676,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
@@ -56905,6 +56927,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -57119,6 +57142,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: ; meta instruction
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:304
@@ -57150,8 +57174,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:176
@@ -57183,8 +57207,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v163, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:48
@@ -57193,7 +57217,6 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
@@ -57233,7 +57256,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_2
@@ -57484,6 +57507,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -62755,8 +62779,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
@@ -62764,7 +62788,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
; SI-NEXT: v_mov_b32_e32 v9, v25
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
@@ -62781,8 +62805,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
@@ -62792,8 +62816,6 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
; SI-NEXT: v_mov_b32_e32 v12, v61
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
@@ -62816,15 +62838,14 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
; SI-NEXT: v_mov_b32_e32 v15, v60
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
@@ -62856,6 +62877,7 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
; SI-NEXT: v_mov_b32_e32 v39, v62
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -62949,9 +62971,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB43_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -74026,8 +74048,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; kill: killed $vgpr36
@@ -74189,7 +74211,7 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; kill: killed $vgpr36
; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -75207,8 +75229,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: ; implicit-def: $vgpr39
; VI-NEXT: ; kill: killed $vgpr39
@@ -75807,11 +75829,11 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v14, v14, v35, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v16, v38, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v35, v35, v48, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -76086,8 +76108,8 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr39
; GFX9-NEXT: ; kill: killed $vgpr39
@@ -76273,13 +76295,14 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB56_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v32
@@ -76725,11 +76748,11 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
@@ -81966,12 +81989,12 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v52
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:28
@@ -81987,20 +82010,21 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:60
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:52
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204
@@ -82010,28 +82034,29 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236
@@ -82041,28 +82066,29 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:268
@@ -82072,28 +82098,29 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:312
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:300
@@ -82103,28 +82130,29 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:344
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
@@ -82134,36 +82162,38 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:376
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:364
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
@@ -83379,9 +83409,9 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:112
@@ -83413,18 +83443,17 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:4
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:288
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -84658,9 +84687,9 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
@@ -84692,10 +84721,10 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_waitcnt vmcnt(31)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -84769,13 +84798,13 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -85960,6 +85989,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
@@ -85991,8 +86021,8 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
@@ -86024,8 +86054,8 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
@@ -86054,7 +86084,6 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
@@ -86092,7 +86121,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -86806,6 +86835,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v86, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:376
@@ -86837,8 +86867,8 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v147, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:268
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:264
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:248
@@ -86870,8 +86900,8 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v59, off, s32 offset:144
; GFX11-FAKE16-NEXT: scratch_load_u16 v60, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:136
-; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v74, off, s32 offset:120
@@ -86900,7 +86930,6 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v120, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v121, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v122, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v123, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v125, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v126, off, s32 offset:8
@@ -86922,7 +86951,7 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v6 :: v_dual_mov_b32 v83, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, v5 :: v_dual_mov_b32 v81, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v2 :: v_dual_mov_b32 v84, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -87769,19 +87798,19 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(9) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:268
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:244
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
@@ -87796,52 +87825,49 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: v_lshlrev_b32_e32 v38, 8, v36
; SI-NEXT: v_lshlrev_b32_e32 v36, 8, v34
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v32
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v49
; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v39
-; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
-; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
-; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
+; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v31
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60
@@ -87853,6 +87879,9 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
@@ -87886,8 +87915,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
@@ -88786,10 +88815,10 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s46, v14
; VI-NEXT: v_readfirstlane_b32 s47, v13
; VI-NEXT: v_readfirstlane_b32 s76, v0
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: v_readfirstlane_b32 s6, v30
; VI-NEXT: v_readfirstlane_b32 s7, v29
@@ -88818,7 +88847,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s73, v3
; VI-NEXT: v_readfirstlane_b32 s74, v2
; VI-NEXT: v_readfirstlane_b32 s75, v1
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -88923,23 +88952,16 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -88960,6 +88982,13 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -88992,7 +89021,6 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
@@ -89000,6 +89028,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
@@ -89673,10 +89702,10 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s46, v14
; GFX9-NEXT: v_readfirstlane_b32 s47, v13
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; GFX9-NEXT: v_readfirstlane_b32 s6, v30
; GFX9-NEXT: v_readfirstlane_b32 s7, v29
@@ -89705,7 +89734,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s73, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
; GFX9-NEXT: v_readfirstlane_b32 s75, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -89810,23 +89839,16 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -89847,6 +89869,13 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -89870,68 +89899,68 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB59_2
@@ -90030,6 +90059,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_or_b32_e32 v16, v17, v16
; GFX9-NEXT: v_perm_b32 v17, v59, v18, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX9-NEXT: s_waitcnt vmcnt(45)
; GFX9-NEXT: v_perm_b32 v18, v41, v40, s4
; GFX9-NEXT: v_or_b32_e32 v17, v18, v17
; GFX9-NEXT: v_perm_b32 v18, v57, v19, s4
@@ -90143,6 +90173,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB59_3
; GFX9-NEXT: .LBB59_2:
+; GFX9-NEXT: s_waitcnt vmcnt(48)
; GFX9-NEXT: v_mov_b32_e32 v57, v56
; GFX9-NEXT: v_mov_b32_e32 v47, v46
; GFX9-NEXT: v_mov_b32_e32 v45, v26
@@ -90555,6 +90586,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:304
@@ -90586,8 +90618,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:176
@@ -90619,8 +90651,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:48
@@ -90629,7 +90661,6 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
@@ -90669,7 +90700,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_2
@@ -90920,6 +90951,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -91134,6 +91166,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: ; meta instruction
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:304
@@ -91165,8 +91198,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:176
@@ -91198,8 +91231,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v163, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:48
@@ -91208,7 +91241,6 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
@@ -91248,7 +91280,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_2
@@ -91499,6 +91531,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -96553,8 +96586,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
@@ -96562,7 +96595,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
; SI-NEXT: v_mov_b32_e32 v9, v25
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
@@ -96579,8 +96612,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
@@ -96590,8 +96623,6 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
; SI-NEXT: v_mov_b32_e32 v12, v61
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
@@ -96614,15 +96645,14 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
; SI-NEXT: v_mov_b32_e32 v15, v60
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
@@ -96654,6 +96684,7 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
; SI-NEXT: v_mov_b32_e32 v39, v62
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -96747,9 +96778,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB63_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -106195,8 +106226,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; SI-NEXT: ; implicit-def: $vgpr36
; SI-NEXT: ; kill: killed $vgpr36
@@ -106358,7 +106389,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr48
; SI-NEXT: ; kill: killed $vgpr36
; SI-NEXT: ; implicit-def: $vgpr36
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -107360,8 +107391,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: ; implicit-def: $vgpr39
; VI-NEXT: ; kill: killed $vgpr39
@@ -107941,9 +107972,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v37, v59, v51, s4
; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v36, v36, v51, s4
; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -108225,8 +108256,8 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr39
; GFX9-NEXT: ; kill: killed $vgpr39
@@ -108409,7 +108440,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: ; implicit-def: $vgpr39
; GFX9-NEXT: ; implicit-def: $vgpr54
; GFX9-NEXT: ; implicit-def: $vgpr58
-; GFX9-NEXT: s_waitcnt vmcnt(25)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: ; kill: killed $vgpr33
@@ -108420,6 +108451,7 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB72_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
+; GFX9-NEXT: s_waitcnt vmcnt(25)
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(25)
@@ -108842,9 +108874,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v39, v49, v50, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v36, v36, v49, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -110752,8 +110784,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_mov_b32_e32 v7, v8
; SI-NEXT: v_mov_b32_e32 v50, v1
; SI-NEXT: v_mov_b32_e32 v39, v56
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
@@ -112414,25 +112446,25 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v26, v26, v45, s4
; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v38, v22, v45, s4
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v31, v31, v22, s4
; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v32, v22, v45, s4
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v27, v27, v22, s4
; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v52, v22, v45, s4
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112446,13 +112478,13 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v19, v19, v23, s4
; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v23, v23, v45, s4
; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:4
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v51
; VI-NEXT: v_or_b32_e32 v29, v48, v29
@@ -112488,7 +112520,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:40
; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(13)
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_perm_b32 v15, v15, v45, s4
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -113630,13 +113662,13 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v27, v27, v19, s4
; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_perm_b32 v45, v59, v52, s4
; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v35, v15, v19, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
@@ -113655,25 +113687,25 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_perm_b32 v20, v28, v15, s4
; GFX9-NEXT: v_perm_b32 v28, v29, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v29, v15, v50, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v23, v23, v15, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v33, v15, v50, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v21, v21, v15, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v50, v15, v50, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -113682,9 +113714,9 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v15, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v45
; GFX9-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:20
@@ -114992,12 +115024,12 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v52
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:28
@@ -115013,20 +115045,21 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:60
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:52
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204
@@ -115036,28 +115069,29 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236
@@ -115067,28 +115101,29 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:244
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:260
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:268
@@ -115098,28 +115133,29 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:312
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:300
@@ -115129,28 +115165,29 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:336
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:344
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
@@ -115160,36 +115197,38 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:376
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:348
; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:364
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
@@ -116405,9 +116444,9 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:112
@@ -116439,18 +116478,17 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:4
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:380
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:288
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -117684,9 +117722,9 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
@@ -117718,10 +117756,10 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_waitcnt vmcnt(31)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; GFX9-NEXT: s_waitcnt vmcnt(32)
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -117795,13 +117833,13 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:376
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:380
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -118986,6 +119024,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:376
@@ -119017,8 +119056,8 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:268
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:260
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:248
@@ -119050,8 +119089,8 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:144
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:120
@@ -119080,7 +119119,6 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v57, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v58, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:388
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v60, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v62, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:8
@@ -119118,7 +119156,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v125.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v126.l, v1.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v127.l, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -119832,6 +119870,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v127, s32 offset:392
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v86, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:376
@@ -119863,8 +119902,8 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v147, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:268
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:264
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:260
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:248
@@ -119896,8 +119935,8 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v59, off, s32 offset:144
; GFX11-FAKE16-NEXT: scratch_load_u16 v60, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:136
-; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v74, off, s32 offset:120
@@ -119926,7 +119965,6 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v120, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v121, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v122, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:388
; GFX11-FAKE16-NEXT: scratch_load_u16 v123, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v125, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v126, off, s32 offset:8
@@ -119948,7 +119986,7 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v6 :: v_dual_mov_b32 v83, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v80, v5 :: v_dual_mov_b32 v81, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v2 :: v_dual_mov_b32 v84, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -120795,19 +120833,19 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; SI-NEXT: s_waitcnt vmcnt(9) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:284
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:268
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:244
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:276
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:260
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
@@ -120822,52 +120860,49 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v38, 8, v36
; SI-NEXT: v_lshlrev_b32_e32 v36, 8, v34
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v32
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:228
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:220
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v49
; SI-NEXT: v_lshlrev_b32_e32 v49, 24, v39
-; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
-; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
-; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:196
+; SI-NEXT: v_lshlrev_b32_e32 v39, 24, v37
+; SI-NEXT: v_lshlrev_b32_e32 v37, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v35, 24, v33
; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v31
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:180
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:164
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:140
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:148
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:132
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:116
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:68
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:60
@@ -120879,6 +120914,9 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:108
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
@@ -120912,8 +120950,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
@@ -121812,10 +121850,10 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s46, v14
; VI-NEXT: v_readfirstlane_b32 s47, v13
; VI-NEXT: v_readfirstlane_b32 s76, v0
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: v_readfirstlane_b32 s6, v30
; VI-NEXT: v_readfirstlane_b32 s7, v29
@@ -121844,7 +121882,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s73, v3
; VI-NEXT: v_readfirstlane_b32 s74, v2
; VI-NEXT: v_readfirstlane_b32 s75, v1
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(4)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -121949,23 +121987,16 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -121986,6 +122017,13 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -122018,7 +122056,6 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
@@ -122026,6 +122063,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
@@ -122699,10 +122737,10 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s46, v14
; GFX9-NEXT: v_readfirstlane_b32 s47, v13
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; GFX9-NEXT: v_readfirstlane_b32 s6, v30
; GFX9-NEXT: v_readfirstlane_b32 s7, v29
@@ -122731,7 +122769,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s73, v3
; GFX9-NEXT: v_readfirstlane_b32 s74, v2
; GFX9-NEXT: v_readfirstlane_b32 s75, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -122836,23 +122874,16 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:164
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:160
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:148
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
-; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:116
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:112
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:104
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:80
@@ -122873,6 +122904,13 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:156
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:144
+; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:140
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:108
+; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
@@ -122896,68 +122934,68 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(44)
+; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(46)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(45)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(43)
+; GFX9-NEXT: s_waitcnt vmcnt(50)
; GFX9-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB75_2
@@ -123056,6 +123094,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: v_or_b32_e32 v16, v17, v16
; GFX9-NEXT: v_perm_b32 v17, v59, v18, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX9-NEXT: s_waitcnt vmcnt(45)
; GFX9-NEXT: v_perm_b32 v18, v41, v40, s4
; GFX9-NEXT: v_or_b32_e32 v17, v18, v17
; GFX9-NEXT: v_perm_b32 v18, v57, v19, s4
@@ -123169,6 +123208,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_branch .LBB75_3
; GFX9-NEXT: .LBB75_2:
+; GFX9-NEXT: s_waitcnt vmcnt(48)
; GFX9-NEXT: v_mov_b32_e32 v57, v56
; GFX9-NEXT: v_mov_b32_e32 v47, v46
; GFX9-NEXT: v_mov_b32_e32 v45, v26
@@ -123581,6 +123621,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:304
@@ -123612,8 +123653,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:176
@@ -123645,8 +123686,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:48
@@ -123655,7 +123696,6 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
@@ -123695,7 +123735,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB75_2
@@ -123946,6 +123986,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -124160,6 +124201,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: ; meta instruction
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:304
@@ -124191,8 +124233,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:176
@@ -124224,8 +124266,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v148, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v161, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v151, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v163, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:48
@@ -124234,7 +124276,6 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
@@ -124274,7 +124315,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB75_2
@@ -124525,6 +124566,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
@@ -129665,8 +129707,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[8:9], v[23:24], 16
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
@@ -129674,7 +129716,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
; SI-NEXT: v_mov_b32_e32 v32, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
; SI-NEXT: v_mov_b32_e32 v9, v25
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
@@ -129691,8 +129733,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[11:12], v[27:28], 16
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
@@ -129702,8 +129744,6 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshrrev_b32_e32 v62, 16, v12
; SI-NEXT: v_mov_b32_e32 v12, v61
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
@@ -129726,15 +129766,14 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v15
; SI-NEXT: v_mov_b32_e32 v15, v60
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
@@ -129766,6 +129805,7 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v30
; SI-NEXT: v_mov_b32_e32 v39, v62
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
@@ -129859,9 +129899,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; SI-NEXT: .LBB79_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
@@ -139385,6 +139425,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:108
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:388
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
@@ -139405,7 +139446,6 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:160
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:176
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:184
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:108
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -139468,7 +139508,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr40
; SI-NEXT: ; implicit-def: $vgpr38
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v17
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -139503,13 +139545,10 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v18
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:104
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v17
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -139592,28 +139631,28 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v28
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:208
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188
; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr28
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v1
-; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(4) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200
@@ -139627,21 +139666,21 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v2
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232
@@ -139657,10 +139696,12 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v0
; SI-NEXT: s_waitcnt vmcnt(2)
@@ -139675,15 +139716,13 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:296
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:292
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v0
; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v5
+; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v0
; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v5
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v43, 8, v11
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:336
@@ -141445,8 +141484,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:112
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:108
@@ -141478,11 +141517,11 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:4
; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -141555,124 +141594,124 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
+; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:188
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:176
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:168
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:164
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:156
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:152
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:148
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:140
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:136
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:132
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -141694,6 +141733,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; VI-NEXT: v_perm_b32 v15, v55, v54, s6
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v51, v50, s6
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; VI-NEXT: ; implicit-def: $vgpr37
@@ -142733,8 +142773,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:32
@@ -142753,7 +142794,6 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:296
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:320
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:160
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
@@ -142762,31 +142802,30 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: s_waitcnt vmcnt(27)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v53, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(25)
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v51, 8, v2
-; GFX9-NEXT: s_waitcnt vmcnt(24)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: v_lshlrev_b16_e32 v49, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(25)
; GFX9-NEXT: v_lshlrev_b16_e32 v39, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(22)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_lshlrev_b16_e32 v38, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(21)
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshlrev_b16_e32 v34, 8, v6
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -142818,24 +142857,24 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v18
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:352
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:332
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:348
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -142843,20 +142882,19 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:344
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:360
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:384
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:364
; GFX9-NEXT: s_nop 0
@@ -142879,6 +142917,8 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:340
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:76
@@ -142895,13 +142935,12 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(16)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:252
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -142942,10 +142981,10 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -143986,39 +144025,52 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
-; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
@@ -144038,6 +144090,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
@@ -144051,23 +144104,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
-; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
@@ -144120,36 +144159,28 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
@@ -144849,39 +144880,52 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, v3 :: v_dual_mov_b32 v35, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v38, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
+; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
+; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v105, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v93, off, s32 offset:376
; GFX11-FAKE16-NEXT: scratch_load_u16 v104, off, s32 offset:372
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v94, off, s32 offset:364
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
; GFX11-FAKE16-NEXT: scratch_load_u16 v106, off, s32 offset:356
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
; GFX11-FAKE16-NEXT: scratch_load_u16 v91, off, s32 offset:348
; GFX11-FAKE16-NEXT: scratch_load_u16 v77, off, s32 offset:344
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v89, off, s32 offset:340
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
; GFX11-FAKE16-NEXT: scratch_load_u16 v88, off, s32 offset:332
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
; GFX11-FAKE16-NEXT: scratch_load_u16 v92, off, s32 offset:324
-; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_u16 v78, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v76, off, s32 offset:308
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:300
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
; GFX11-FAKE16-NEXT: scratch_load_u16 v79, off, s32 offset:292
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:284
; GFX11-FAKE16-NEXT: scratch_load_u16 v45, off, s32 offset:280
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:276
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v58, off, s32 offset:268
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
; GFX11-FAKE16-NEXT: scratch_load_u16 v72, off, s32 offset:260
-; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v41, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v44, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v57, off, s32 offset:248
@@ -144901,6 +144945,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v144, off, s32 offset:192
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:184
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v164, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:176
; GFX11-FAKE16-NEXT: scratch_load_u16 v133, off, s32 offset:172
@@ -144914,23 +144959,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:136
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:132
-; GFX11-FAKE16-NEXT: s_clause 0x1f
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v116, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
-; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
-; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
-; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
-; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:108
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:92
@@ -144954,44 +144985,31 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v29 :: v_dual_mov_b32 v80, v27
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v26 :: v_dual_mov_b32 v81, v25
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v182, 8, v14
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v166, 8, v15
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v180, 8, v16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v46, 8, v17
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v47, 8, v18
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v56, 8, v19
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v75, 8, v20
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v59, 8, v21
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v74, 8, v22
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v95, 8, v23
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v90, 8, v24
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -145279,6 +145297,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(60)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v76, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v62
; GFX11-FAKE16-NEXT: v_add_nc_u16 v29, 0x300, v0
@@ -145292,6 +145311,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v122, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(58)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v79, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v73, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v73, 0x300, v0
@@ -145299,7 +145319,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v121, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v61, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(53)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v72, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v61, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v63, 3
@@ -145314,7 +145336,9 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v110, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v58, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(49)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v60, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(46)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v42
; GFX11-FAKE16-NEXT: v_add_nc_u16 v25, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v58, 0x300, v0
@@ -145324,6 +145348,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v57
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v24, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(45)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v43, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v108, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
@@ -145337,6 +145362,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v181
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(41)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v183, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v181, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v179, 3
@@ -145346,6 +145372,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v178
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v176
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v177, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v176, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v165, 3
@@ -145355,6 +145382,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v163
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v161
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v6, v164, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v21, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u16 v161, 0x300, v1
@@ -145364,6 +145392,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v160
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v149, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v20, 0x300, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v147, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v147, 0x300, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v2
@@ -145373,6 +145402,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v146
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v133, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v19, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v131, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
@@ -145387,6 +145417,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v102
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v103, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v102, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v98, 3
@@ -145396,6 +145427,7 @@ define <64 x bfloat> @bitcast_v128i8_to_v64bf16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v97
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v96
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v115, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v96, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v116, 3
@@ -147269,8 +147301,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; VI-NEXT: v_readfirstlane_b32 s63, v12
; VI-NEXT: v_readfirstlane_b32 s8, v0
-; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:276
+; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: v_readfirstlane_b32 s45, v11
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:12
@@ -147302,9 +147334,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: v_readfirstlane_b32 s7, v3
; VI-NEXT: v_readfirstlane_b32 s11, v2
; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:4
-; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:324
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
@@ -147319,10 +147351,10 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:284
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
; VI-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:272
; VI-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:268
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:260
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -147330,9 +147362,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:248
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -147357,12 +147389,20 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:212
+; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:208
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
+; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
+; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
+; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
+; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
+; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:200
; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:196
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:192
@@ -147377,41 +147417,35 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:140
-; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
-; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
-; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
-; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
+; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:88
+; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:80
; VI-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:76
-; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
-; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
+; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
+; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
+; VI-NEXT: s_waitcnt vmcnt(11)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
+; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:56
-; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
; VI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
@@ -147445,23 +147479,22 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:40
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:32
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:24
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; VI-NEXT: s_cbranch_scc0 .LBB89_2
; VI-NEXT: ; %bb.1: ; %cmp.false
@@ -148167,11 +148200,11 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
; GFX9-NEXT: v_readfirstlane_b32 s43, v2
@@ -148181,6 +148214,11 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -148198,21 +148236,19 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
@@ -148222,8 +148258,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -148252,10 +148287,10 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -148263,66 +148298,66 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(33)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -148347,7 +148382,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
@@ -148373,7 +148408,6 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB89_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
@@ -148485,8 +148519,10 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: s_waitcnt vmcnt(56)
; GFX9-NEXT: v_mov_b32_e32 v53, v59
; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(55)
; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
@@ -148615,6 +148651,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: v_mov_b32_e32 v63, v39
; GFX9-NEXT: v_mov_b32_e32 v38, v36
; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: s_waitcnt vmcnt(51)
; GFX9-NEXT: v_mov_b32_e32 v50, v59
; GFX9-NEXT: v_mov_b32_e32 v35, v60
; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
@@ -149052,6 +149089,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
@@ -149083,8 +149121,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
@@ -149116,8 +149154,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
@@ -149126,7 +149164,6 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
@@ -149166,7 +149203,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB89_2
@@ -149356,15 +149393,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
@@ -149373,12 +149413,14 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
@@ -149386,15 +149428,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
@@ -149403,12 +149448,14 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
@@ -149416,15 +149463,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
@@ -149433,6 +149483,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
@@ -149573,6 +149624,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s32 offset:320 ; 4-byte Folded Spill
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:304
@@ -149604,8 +149656,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v130, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v129, off, s32 offset:176
@@ -149637,8 +149689,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
@@ -149647,7 +149699,6 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
@@ -149687,7 +149738,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB89_2
@@ -149915,15 +149966,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v150, 0x300, v4
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v151, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v162
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v135, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(57)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v148
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v145, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v133, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v144, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v132
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v135, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
@@ -149943,11 +149997,13 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v115, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v118
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v113
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v99, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v117, 0xc0c0004
@@ -149955,12 +150011,14 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v100, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v99, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v114, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v87
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v84, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v103, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v2
@@ -149973,17 +150031,21 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v71
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
@@ -152219,8 +152281,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: ; implicit-def: $vgpr39
; VI-NEXT: ; kill: killed $vgpr39
@@ -153640,9 +153702,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:8
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: ; kill: killed $vgpr33
@@ -153832,7 +153894,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(29)
+; GFX9-NEXT: s_waitcnt vmcnt(30)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: ; implicit-def: $vgpr31
; GFX9-NEXT: ; kill: killed $vgpr31
@@ -154288,6 +154350,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
+; GFX9-NEXT: s_waitcnt vmcnt(56)
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v61
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; GFX9-NEXT: v_cndmask_b32_e32 v55, v18, v19, vcc
@@ -154860,16 +154923,16 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v36
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v36
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v35
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(15)
; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[15:16]
; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
@@ -154885,14 +154948,15 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshrrev_b32_e32 v32, 8, v55
; GFX9-NEXT: v_lshrrev_b32_e32 v40, 8, v43
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 8, v42
-; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: s_waitcnt vmcnt(10)
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 24, v36
; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(10)
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v35
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 24, v34
; GFX9-NEXT: v_lshrrev_b32_e32 v36, 8, v36
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v39
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
@@ -154901,6 +154965,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v39
+; GFX9-NEXT: s_waitcnt vmcnt(11)
; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v38
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v63
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 24, v36
@@ -165795,6 +165860,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:388
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
@@ -165814,7 +165880,6 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144
; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:184
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92
; SI-NEXT: v_lshlrev_b32_e32 v40, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v9
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
@@ -165824,9 +165889,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v25
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4
; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165881,7 +165946,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v26
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -165913,21 +165980,19 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v26
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:116
-; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
@@ -165937,45 +166002,47 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:28
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:100
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:100
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v8
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v5
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v28
@@ -165988,34 +166055,31 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v5
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v10
-; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v32
@@ -166026,194 +166090,196 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v34
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:196
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v10
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v11
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:208
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:280
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v3
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(7) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:232
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v12
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v12
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v63, 8, v2
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:260
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:300
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304
+; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v0
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v13
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:292
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v3
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v14
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v15
; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:376
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v0
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:356
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:360
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:364
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:360
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v0
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:384
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v18
; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v62, 24, v19
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v20
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v15
; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60
@@ -167857,8 +167923,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:112
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:108
@@ -167890,11 +167956,11 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:4
; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -167967,124 +168033,124 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
+; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:188
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:176
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:168
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:164
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:156
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:152
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:148
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:140
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:136
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:132
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -168106,6 +168172,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; VI-NEXT: v_perm_b32 v15, v55, v54, s6
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v51, v50, s6
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; VI-NEXT: ; implicit-def: $vgpr37
@@ -169145,8 +169212,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:32
@@ -169165,7 +169233,6 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:296
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:320
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:160
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
@@ -169174,31 +169241,30 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: s_waitcnt vmcnt(27)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v53, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(25)
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v51, 8, v2
-; GFX9-NEXT: s_waitcnt vmcnt(24)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: v_lshlrev_b16_e32 v49, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(25)
; GFX9-NEXT: v_lshlrev_b16_e32 v39, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(22)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_lshlrev_b16_e32 v38, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(21)
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshlrev_b16_e32 v34, 8, v6
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -169230,24 +169296,24 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v18
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:352
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:332
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:348
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -169255,20 +169321,19 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:344
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:360
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:384
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:364
; GFX9-NEXT: s_nop 0
@@ -169291,6 +169356,8 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:340
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:76
@@ -169307,13 +169374,12 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(16)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:252
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -169354,10 +169420,10 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -170398,39 +170464,52 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
-; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
@@ -170450,6 +170529,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
@@ -170463,23 +170543,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
-; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
@@ -170532,36 +170598,28 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
@@ -171261,39 +171319,52 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, v3 :: v_dual_mov_b32 v35, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v38, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
+; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
+; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v105, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v93, off, s32 offset:376
; GFX11-FAKE16-NEXT: scratch_load_u16 v104, off, s32 offset:372
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v94, off, s32 offset:364
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
; GFX11-FAKE16-NEXT: scratch_load_u16 v106, off, s32 offset:356
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
; GFX11-FAKE16-NEXT: scratch_load_u16 v91, off, s32 offset:348
; GFX11-FAKE16-NEXT: scratch_load_u16 v77, off, s32 offset:344
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v89, off, s32 offset:340
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
; GFX11-FAKE16-NEXT: scratch_load_u16 v88, off, s32 offset:332
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
; GFX11-FAKE16-NEXT: scratch_load_u16 v92, off, s32 offset:324
-; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_u16 v78, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v76, off, s32 offset:308
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:300
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
; GFX11-FAKE16-NEXT: scratch_load_u16 v79, off, s32 offset:292
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:284
; GFX11-FAKE16-NEXT: scratch_load_u16 v45, off, s32 offset:280
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:276
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v58, off, s32 offset:268
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
; GFX11-FAKE16-NEXT: scratch_load_u16 v72, off, s32 offset:260
-; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v41, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v44, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v57, off, s32 offset:248
@@ -171313,6 +171384,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v144, off, s32 offset:192
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:184
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v164, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:176
; GFX11-FAKE16-NEXT: scratch_load_u16 v133, off, s32 offset:172
@@ -171326,23 +171398,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:136
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:132
-; GFX11-FAKE16-NEXT: s_clause 0x1f
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v116, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
-; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
-; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
-; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
-; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:108
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:92
@@ -171366,44 +171424,31 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v29 :: v_dual_mov_b32 v80, v27
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v26 :: v_dual_mov_b32 v81, v25
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v182, 8, v14
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v166, 8, v15
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v180, 8, v16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v46, 8, v17
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v47, 8, v18
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v56, 8, v19
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v75, 8, v20
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v59, 8, v21
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v74, 8, v22
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v95, 8, v23
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v90, 8, v24
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -171691,6 +171736,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(60)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v76, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v62
; GFX11-FAKE16-NEXT: v_add_nc_u16 v29, 0x300, v0
@@ -171704,6 +171750,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v122, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(58)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v79, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v73, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v73, 0x300, v0
@@ -171711,7 +171758,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v121, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v61, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(53)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v72, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v61, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v63, 3
@@ -171726,7 +171775,9 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v110, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v58, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(49)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v60, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(46)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v42
; GFX11-FAKE16-NEXT: v_add_nc_u16 v25, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v58, 0x300, v0
@@ -171736,6 +171787,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v57
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v24, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(45)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v43, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v108, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
@@ -171749,6 +171801,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v181
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(41)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v183, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v181, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v179, 3
@@ -171758,6 +171811,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v178
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v176
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v177, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v176, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v165, 3
@@ -171767,6 +171821,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v163
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v161
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v6, v164, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v21, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u16 v161, 0x300, v1
@@ -171776,6 +171831,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v160
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v149, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v20, 0x300, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v147, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v147, 0x300, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v2
@@ -171785,6 +171841,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v146
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v133, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v19, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v131, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
@@ -171799,6 +171856,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v102
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v103, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v102, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v98, 3
@@ -171808,6 +171866,7 @@ define <64 x half> @bitcast_v128i8_to_v64f16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v97
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v96
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v115, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v96, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v116, 3
@@ -173686,8 +173745,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; VI-NEXT: v_readfirstlane_b32 s63, v12
; VI-NEXT: v_readfirstlane_b32 s8, v0
-; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:276
+; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: v_readfirstlane_b32 s45, v11
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:12
@@ -173719,9 +173778,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: v_readfirstlane_b32 s7, v3
; VI-NEXT: v_readfirstlane_b32 s11, v2
; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:4
-; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:324
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
@@ -173736,10 +173795,10 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:284
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
; VI-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:272
; VI-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:268
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:260
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -173747,9 +173806,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:248
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -173774,12 +173833,20 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:212
+; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:208
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
+; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
+; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
+; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
+; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
+; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:200
; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:196
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:192
@@ -173794,41 +173861,35 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:140
-; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
-; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
-; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
-; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
+; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:88
+; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:80
; VI-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:76
-; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
-; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
+; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
+; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
+; VI-NEXT: s_waitcnt vmcnt(11)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
+; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:56
-; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
; VI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
@@ -173862,23 +173923,22 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:40
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:32
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:24
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; VI-NEXT: s_cbranch_scc0 .LBB93_2
; VI-NEXT: ; %bb.1: ; %cmp.false
@@ -174584,11 +174644,11 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
; GFX9-NEXT: v_readfirstlane_b32 s43, v2
@@ -174598,6 +174658,11 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -174615,21 +174680,19 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
@@ -174639,8 +174702,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -174669,10 +174731,10 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -174680,66 +174742,66 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(33)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -174764,7 +174826,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
@@ -174790,7 +174852,6 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB93_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
@@ -174902,8 +174963,10 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: s_waitcnt vmcnt(56)
; GFX9-NEXT: v_mov_b32_e32 v53, v59
; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(55)
; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
@@ -175032,6 +175095,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: v_mov_b32_e32 v63, v39
; GFX9-NEXT: v_mov_b32_e32 v38, v36
; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: s_waitcnt vmcnt(51)
; GFX9-NEXT: v_mov_b32_e32 v50, v59
; GFX9-NEXT: v_mov_b32_e32 v35, v60
; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
@@ -175469,6 +175533,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
@@ -175500,8 +175565,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
@@ -175533,8 +175598,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
@@ -175543,7 +175608,6 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
@@ -175583,7 +175647,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB93_2
@@ -175773,15 +175837,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
@@ -175790,12 +175857,14 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
@@ -175803,15 +175872,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
@@ -175820,12 +175892,14 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
@@ -175833,15 +175907,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
@@ -175850,6 +175927,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
@@ -175990,6 +176068,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s32 offset:320 ; 4-byte Folded Spill
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:304
@@ -176021,8 +176100,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v130, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v129, off, s32 offset:176
@@ -176054,8 +176133,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
@@ -176064,7 +176143,6 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
@@ -176104,7 +176182,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB93_2
@@ -176332,15 +176410,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v150, 0x300, v4
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v151, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v162
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v135, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(57)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v148
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v145, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v133, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v144, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v132
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v135, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
@@ -176360,11 +176441,13 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v115, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v118
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v113
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v99, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v117, 0xc0c0004
@@ -176372,12 +176455,14 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v100, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v99, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v114, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v87
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v84, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v103, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v2
@@ -176390,17 +176475,21 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v71
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
@@ -178730,8 +178819,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_cbranch_execz .LBB94_2
; VI-NEXT: ; %bb.1: ; %cmp.false
-; VI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: v_lshrrev_b32_e32 v3, 24, v16
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v16
@@ -178893,8 +178982,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
; VI-NEXT: v_add_f16_sdwa v3, v20, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; VI-NEXT: v_add_f16_e32 v20, 0x200, v20
; VI-NEXT: v_add_f16_sdwa v4, v24, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -178921,13 +179010,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
-; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: v_add_f16_sdwa v3, v22, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v22, 0x200, v22
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: v_or_b32_e32 v55, v19, v29
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
; VI-NEXT: v_or_b32_e32 v58, v22, v29
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: v_add_f16_sdwa v29, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, 0x200, v21
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
@@ -178937,8 +179027,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_add_f16_sdwa v3, v26, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v26, 0x200, v26
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: v_or_b32_e32 v57, v21, v29
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
; VI-NEXT: v_add_f16_sdwa v21, v25, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
@@ -178954,18 +179044,19 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v60, v28, v29
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v21
-; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
; VI-NEXT: v_add_f16_e32 v27, 0x200, v27
; VI-NEXT: v_or_b32_e32 v59, v27, v29
; VI-NEXT: v_mov_b32_e32 v16, v27
; VI-NEXT: v_mov_b32_e32 v17, v28
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_add_f16_sdwa v3, v22, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
; VI-NEXT: v_add_f16_e32 v22, 0x200, v22
; VI-NEXT: v_or_b32_e32 v37, v22, v29
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_add_f16_sdwa v29, v21, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v21, 0x200, v21
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
@@ -179049,13 +179140,13 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v3
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; VI-NEXT: v_or_b32_e32 v31, v19, v29
-; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_add_f16_sdwa v3, v19, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_f16_sdwa v35, v20, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v20, 0x200, v20
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_add_f16_sdwa v3, v19, v44 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; VI-NEXT: v_add_f16_e32 v19, 0x200, v19
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v35
; VI-NEXT: v_or_b32_e32 v30, v20, v29
@@ -179162,29 +179253,29 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_bfe_u32 v3, v18, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v22, 8, 8
+; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v43
+; VI-NEXT: v_lshrrev_b64 v[43:44], 24, v[42:43]
+; VI-NEXT: v_lshrrev_b64 v[55:56], 24, v[55:56]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v63, 8, 8
; VI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v13, 8, 8
; VI-NEXT: v_lshrrev_b32_e32 v5, 8, v41
; VI-NEXT: v_lshrrev_b32_e32 v31, 8, v40
; VI-NEXT: v_lshrrev_b64 v[40:41], 24, v[40:41]
-; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v43
-; VI-NEXT: v_lshrrev_b64 v[43:44], 24, v[42:43]
-; VI-NEXT: v_lshrrev_b64 v[55:56], 24, v[55:56]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
; VI-NEXT: v_lshrrev_b64 v[41:42], 24, v[45:46]
; VI-NEXT: v_lshrrev_b32_e32 v62, 8, v46
+; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
@@ -179206,14 +179297,15 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_bfe_u32 v3, v3, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(13)
; VI-NEXT: v_bfe_u32 v3, v56, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(11)
+; VI-NEXT: s_waitcnt vmcnt(13)
; VI-NEXT: v_bfe_u32 v3, v38, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(13)
; VI-NEXT: v_bfe_u32 v3, v42, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(13)
; VI-NEXT: v_bfe_u32 v3, v46, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v44, 8, 8
@@ -179285,9 +179377,9 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v49, v13, v14, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v13, v13, v15, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -179298,9 +179390,9 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v50, v15, v21, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v15, v15, v21, s4
; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -179404,9 +179496,9 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v1, v2, v1, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
@@ -179464,9 +179556,9 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v1, v2, v1, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
@@ -179562,8 +179654,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr34
; GFX9-NEXT: ; kill: killed $vgpr34
@@ -179739,7 +179831,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(13)
+; GFX9-NEXT: s_waitcnt vmcnt(14)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -179755,6 +179847,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v15
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v14
+; GFX9-NEXT: s_waitcnt vmcnt(17)
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 24, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v14
@@ -179927,6 +180020,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: s_cbranch_execz .LBB94_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
; GFX9-NEXT: s_movk_i32 s6, 0x200
+; GFX9-NEXT: s_waitcnt vmcnt(13)
; GFX9-NEXT: v_pk_add_f16 v32, v32, s6 op_sel_hi:[1,0]
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_pk_add_f16 v31, v31, s6 op_sel_hi:[1,0]
@@ -183793,23 +183887,14 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_lshrrev_b64 v[18:19], 24, v[19:20]
; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v28
; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v27
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v58, 8, 8
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v26, v42, 8, 8
; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v31
; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[31:32]
-; VI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v54
-; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[54:55]
-; VI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; VI-NEXT: v_bfe_u32 v15, v5, 8, 8
-; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; VI-NEXT: v_bfe_u32 v2, v14, 8, 8
-; VI-NEXT: v_mov_b32_e32 v5, v35
-; VI-NEXT: v_bfe_u32 v21, v35, 8, 8
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v28, v39
; VI-NEXT: v_bfe_u32 v22, v39, 8, 8
; VI-NEXT: v_mov_b32_e32 v31, v36
@@ -183820,15 +183905,24 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_bfe_u32 v3, v60, 8, 8
; VI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v54, v50
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: v_mov_b32_e32 v39, v13
; VI-NEXT: v_mov_b32_e32 v13, v48
; VI-NEXT: v_mov_b32_e32 v48, v9
; VI-NEXT: v_mov_b32_e32 v9, v41
; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; VI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v54
+; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[54:55]
+; VI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; VI-NEXT: v_bfe_u32 v15, v5, 8, 8
+; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; VI-NEXT: v_bfe_u32 v2, v14, 8, 8
+; VI-NEXT: v_mov_b32_e32 v5, v35
+; VI-NEXT: v_bfe_u32 v21, v35, 8, 8
+; VI-NEXT: v_mov_b32_e32 v54, v50
+; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
; VI-NEXT: v_lshrrev_b32_e32 v25, 8, v38
; VI-NEXT: v_lshrrev_b32_e32 v20, 8, v37
@@ -183841,7 +183935,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_bfe_u32 v24, v7, 8, 8
; VI-NEXT: v_bfe_u32 v32, v51, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(6)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_bfe_u32 v12, v60, 8, 8
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_bfe_u32 v26, v50, 8, 8
@@ -184124,6 +184218,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; VI-NEXT: v_perm_b32 v17, v56, v17, s4
; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v14, v14, v37, s4
; VI-NEXT: v_readlane_b32 s30, v63, 30
; VI-NEXT: v_readlane_b32 s31, v63, 31
@@ -185422,9 +185517,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v28, v51, v52, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v26, v15, v51, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -185433,9 +185528,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v20, v20, v15, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
; GFX9-NEXT: buffer_store_dword v39, v0, s[0:3], 0 offen
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
@@ -186623,6 +186718,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:388
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
@@ -186642,7 +186738,6 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:160
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144
; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:184
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92
; SI-NEXT: v_lshlrev_b32_e32 v40, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v9
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
@@ -186652,9 +186747,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v25
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:4
; SI-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -186709,7 +186804,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr37
; SI-NEXT: ; implicit-def: $vgpr45
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v26
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -186741,21 +186838,19 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v24
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v26
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:116
-; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76
@@ -186765,45 +186860,47 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:28
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:100
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:100
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v6
+; SI-NEXT: ; implicit-def: $vgpr6
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
-; SI-NEXT: ; implicit-def: $vgpr6
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:136
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v8
+; SI-NEXT: ; implicit-def: $vgpr8
+; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v5
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v28
@@ -186816,34 +186913,31 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v5
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr8
; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr31
; SI-NEXT: ; implicit-def: $vgpr30
; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:176
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:168
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v10
-; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v32
@@ -186854,194 +186948,196 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v34
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:720 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:216
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:200
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:196
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v10
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr10
; SI-NEXT: ; implicit-def: $vgpr34
; SI-NEXT: ; implicit-def: $vgpr33
; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v3
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:724 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v11
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:208
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:280
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:252
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v3
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:232
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(7) expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v2
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:232
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:228
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:312
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v12
; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v12
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v63, 8, v2
; SI-NEXT: ; implicit-def: $vgpr12
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(3) expcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:260
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:264
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:340
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:728 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:300
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304
+; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v0
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v13
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr13
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:332
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:292
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v7, 24, v3
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v14
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v15
; SI-NEXT: ; implicit-def: $vgpr14
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:352
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:376
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:348
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v0
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:384
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380
-; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:356
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:360
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:372
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:368
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:364
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:360
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:356
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v0
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:384
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:380
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v18
; SI-NEXT: ; implicit-def: $vgpr18
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v15
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v62, 24, v19
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v20
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v15
; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $vgpr21
; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: ; implicit-def: $vgpr20
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60
@@ -188685,8 +188781,8 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32
+; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:116
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:112
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:108
@@ -188718,11 +188814,11 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:4
; VI-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:296
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:384
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -188795,124 +188891,124 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:372
+; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:376
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:364
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:360
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:188
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:356
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:352
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:348
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:176
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:344
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:340
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:168
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:164
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:332
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:328
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:156
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:324
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:152
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:148
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:316
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:312
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:140
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:308
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:136
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:304
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:132
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:300
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -188934,6 +189030,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; VI-NEXT: v_perm_b32 v15, v55, v54, s6
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_perm_b32 v16, v51, v50, s6
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; VI-NEXT: ; implicit-def: $vgpr37
@@ -189973,8 +190070,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:32
@@ -189993,7 +190091,6 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:296
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:304
; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:320
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:160
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
@@ -190002,31 +190099,30 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:132
-; GFX9-NEXT: s_waitcnt vmcnt(27)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v53, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(25)
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
+; GFX9-NEXT: s_waitcnt vmcnt(27)
; GFX9-NEXT: v_lshlrev_b16_e32 v51, 8, v2
-; GFX9-NEXT: s_waitcnt vmcnt(24)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: v_lshlrev_b16_e32 v49, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(23)
+; GFX9-NEXT: s_waitcnt vmcnt(25)
; GFX9-NEXT: v_lshlrev_b16_e32 v39, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(22)
+; GFX9-NEXT: s_waitcnt vmcnt(24)
; GFX9-NEXT: v_lshlrev_b16_e32 v38, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(21)
+; GFX9-NEXT: s_waitcnt vmcnt(23)
; GFX9-NEXT: v_lshlrev_b16_e32 v34, 8, v6
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:292
-; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:100
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:92
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:316
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -190058,24 +190154,24 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v18
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:328
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:336
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:352
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:324
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:332
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:348
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -190083,20 +190179,19 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:344
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:360
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:368
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:384
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:356
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v2
; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:364
; GFX9-NEXT: s_nop 0
@@ -190119,6 +190214,8 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:340
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:76
@@ -190135,13 +190232,12 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:124
; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:120
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:372
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(16)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:252
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -190182,10 +190278,10 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:740 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:200
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -191226,39 +191322,52 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:392
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:380
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v75, off, s32 offset:376
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v78, off, s32 offset:372
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:368
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:364
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:360
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:356
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:352
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:348
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v46, off, s32 offset:344
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v74, off, s32 offset:340
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:336
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:332
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:328
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:324
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:320
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v61, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v72, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:296
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:292
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:284
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v183, off, s32 offset:280
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v59, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:260
-; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v44, off, s32 offset:252
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:248
@@ -191278,6 +191387,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:192
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:184
+; GFX11-TRUE16-NEXT: s_clause 0x1f
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:176
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:172
@@ -191291,23 +191401,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:136
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:132
-; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:388
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:92
@@ -191360,36 +191456,28 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v88
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v52.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v52.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v54.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v64.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v71.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v71.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v65.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v66.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v66.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.h, 8, v67.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v67.l, 8, v67.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v69.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v70.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.l, 8, v70.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v31.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v71.h, 8, v31.l
@@ -192089,39 +192177,52 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v34, v3 :: v_dual_mov_b32 v35, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v38, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
+; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
+; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:384
; GFX11-FAKE16-NEXT: scratch_load_u16 v105, off, s32 offset:380
; GFX11-FAKE16-NEXT: scratch_load_u16 v93, off, s32 offset:376
; GFX11-FAKE16-NEXT: scratch_load_u16 v104, off, s32 offset:372
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:368
; GFX11-FAKE16-NEXT: scratch_load_u16 v94, off, s32 offset:364
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:360
; GFX11-FAKE16-NEXT: scratch_load_u16 v106, off, s32 offset:356
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:352
; GFX11-FAKE16-NEXT: scratch_load_u16 v91, off, s32 offset:348
; GFX11-FAKE16-NEXT: scratch_load_u16 v77, off, s32 offset:344
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v89, off, s32 offset:340
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:336
; GFX11-FAKE16-NEXT: scratch_load_u16 v88, off, s32 offset:332
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:328
; GFX11-FAKE16-NEXT: scratch_load_u16 v92, off, s32 offset:324
-; GFX11-FAKE16-NEXT: scratch_load_u16 v6, off, s32 offset:320
; GFX11-FAKE16-NEXT: scratch_load_u16 v78, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v62, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v76, off, s32 offset:308
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:304
; GFX11-FAKE16-NEXT: scratch_load_u16 v73, off, s32 offset:300
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:296
; GFX11-FAKE16-NEXT: scratch_load_u16 v79, off, s32 offset:292
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:288
; GFX11-FAKE16-NEXT: scratch_load_u16 v63, off, s32 offset:284
; GFX11-FAKE16-NEXT: scratch_load_u16 v45, off, s32 offset:280
; GFX11-FAKE16-NEXT: scratch_load_u16 v61, off, s32 offset:276
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:272
; GFX11-FAKE16-NEXT: scratch_load_u16 v58, off, s32 offset:268
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:264
; GFX11-FAKE16-NEXT: scratch_load_u16 v72, off, s32 offset:260
-; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v41, off, s32 offset:256
; GFX11-FAKE16-NEXT: scratch_load_u16 v44, off, s32 offset:252
; GFX11-FAKE16-NEXT: scratch_load_u16 v57, off, s32 offset:248
@@ -192141,6 +192242,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v144, off, s32 offset:192
; GFX11-FAKE16-NEXT: scratch_load_u16 v149, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v160, off, s32 offset:184
+; GFX11-FAKE16-NEXT: s_clause 0x1f
; GFX11-FAKE16-NEXT: scratch_load_u16 v164, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:176
; GFX11-FAKE16-NEXT: scratch_load_u16 v133, off, s32 offset:172
@@ -192154,23 +192256,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:140
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:136
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:132
-; GFX11-FAKE16-NEXT: s_clause 0x1f
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v116, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_b32 v13, off, s32 offset:388
-; GFX11-FAKE16-NEXT: scratch_load_u16 v14, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v15, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v16, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v17, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v18, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v19, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v20, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v21, off, s32 offset:80
-; GFX11-FAKE16-NEXT: scratch_load_u16 v22, off, s32 offset:96
-; GFX11-FAKE16-NEXT: scratch_load_u16 v23, off, s32 offset:104
-; GFX11-FAKE16-NEXT: scratch_load_u16 v24, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:108
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v150, off, s32 offset:92
@@ -192194,44 +192282,31 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v85, v29 :: v_dual_mov_b32 v80, v27
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v82, v26 :: v_dual_mov_b32 v81, v25
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v182, 8, v14
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v166, 8, v15
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v180, 8, v16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v46, 8, v17
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v47, 8, v18
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v56, 8, v19
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v75, 8, v20
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v59, 8, v21
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(20)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v74, 8, v22
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v95, 8, v23
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v90, 8, v24
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v107, 8, v12
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v109, 8, v11
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v108, 8, v10
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v110, 8, v9
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v111, 8, v8
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v120, 8, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v121, 8, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v124, 8, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v122, 8, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v123, 8, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v125, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v126, 8, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v127, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -192519,6 +192594,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(60)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v76, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v62
; GFX11-FAKE16-NEXT: v_add_nc_u16 v29, 0x300, v0
@@ -192532,6 +192608,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v122, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(58)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v79, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v73, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v73, 0x300, v0
@@ -192539,7 +192616,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v121, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v61, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(53)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v72, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v61, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v63, 3
@@ -192554,7 +192633,9 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v110, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v58, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(49)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v60, 3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(46)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v42
; GFX11-FAKE16-NEXT: v_add_nc_u16 v25, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v58, 0x300, v0
@@ -192564,6 +192645,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v57
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v24, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(45)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v43, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v108, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
@@ -192577,6 +192659,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v181
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(41)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v183, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v181, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v179, 3
@@ -192586,6 +192669,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v178
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v176
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v177, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v176, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v165, 3
@@ -192595,6 +192679,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v163
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v161
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v6, v164, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v21, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u16 v161, 0x300, v1
@@ -192604,6 +192689,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v160
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v149, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v20, 0x300, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v147, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v147, 0x300, v0
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v2
@@ -192613,6 +192699,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v146
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v133, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v19, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v131, 3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
@@ -192627,6 +192714,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v5, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v6
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v102
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v103, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v102, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v98, 3
@@ -192636,6 +192724,7 @@ define <64 x i16> @bitcast_v128i8_to_v64i16(<128 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v97
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v96
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v115, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v96, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v116, 3
@@ -194514,8 +194603,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; VI-NEXT: v_readfirstlane_b32 s63, v12
; VI-NEXT: v_readfirstlane_b32 s8, v0
-; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:276
+; VI-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:280
; VI-NEXT: v_readfirstlane_b32 s45, v11
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:12
@@ -194547,9 +194636,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s7, v3
; VI-NEXT: v_readfirstlane_b32 s11, v2
; VI-NEXT: v_readfirstlane_b32 s6, v1
+; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:4
-; VI-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
; VI-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:324
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
@@ -194564,10 +194653,10 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:284
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
; VI-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:272
; VI-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:268
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:264
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:260
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -194575,9 +194664,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:256
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:248
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:252
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -194602,12 +194691,20 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
; VI-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:212
+; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:208
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:204
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
+; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
+; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
+; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
+; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
+; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:200
; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:196
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:192
@@ -194622,41 +194719,35 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:156
; VI-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:152
; VI-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:148
-; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:144
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:140
-; VI-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:136
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:132
-; VI-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:120
-; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:116
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:112
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
-; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
-; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
-; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
+; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
; VI-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:88
+; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:80
; VI-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:76
-; VI-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:72
-; VI-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:68
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:64
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:104
+; VI-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:100
+; VI-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:96
+; VI-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:92
+; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:108
+; VI-NEXT: s_waitcnt vmcnt(11)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
+; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
; VI-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:56
-; VI-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:52
-; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:48
; VI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(5)
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
@@ -194690,23 +194781,22 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:40
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:44
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:32
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:36
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:24
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:28
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:16
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:20
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; VI-NEXT: s_cbranch_scc0 .LBB97_2
; VI-NEXT: ; %bb.1: ; %cmp.false
@@ -195412,11 +195502,11 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s61, v11
; GFX9-NEXT: v_readfirstlane_b32 s15, v3
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: buffer_load_ushort v11, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_ushort v12, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_ushort v13, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:328
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:324
; GFX9-NEXT: v_readfirstlane_b32 s8, v5
; GFX9-NEXT: v_readfirstlane_b32 s10, v4
; GFX9-NEXT: v_readfirstlane_b32 s43, v2
@@ -195426,6 +195516,11 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s14, v8
; GFX9-NEXT: v_readfirstlane_b32 s47, v7
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:332
+; GFX9-NEXT: v_readfirstlane_b32 s59, v6
+; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
+; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
+; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
+; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
; GFX9-NEXT: v_readfirstlane_b32 s74, v30
; GFX9-NEXT: v_readfirstlane_b32 s44, v29
; GFX9-NEXT: v_readfirstlane_b32 s56, v28
@@ -195443,21 +195538,19 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s58, v16
; GFX9-NEXT: v_readfirstlane_b32 s73, v15
; GFX9-NEXT: v_readfirstlane_b32 s75, v14
-; GFX9-NEXT: v_readfirstlane_b32 s59, v6
-; GFX9-NEXT: buffer_load_ushort v39, off, s[0:3], s32 offset:296
-; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:292
-; GFX9-NEXT: buffer_load_ushort v62, off, s[0:3], s32 offset:288
-; GFX9-NEXT: buffer_load_ushort v6, off, s[0:3], s32 offset:284
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:320
; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:316
; GFX9-NEXT: buffer_load_ushort v4, off, s[0:3], s32 offset:312
; GFX9-NEXT: buffer_load_ushort v5, off, s[0:3], s32 offset:308
; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:304
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:300
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v63, off, s[0:3], s32 offset:280
; GFX9-NEXT: buffer_load_ushort v38, off, s[0:3], s32 offset:276
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:272
@@ -195467,8 +195560,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v9, off, s[0:3], s32 offset:256
; GFX9-NEXT: buffer_load_ushort v37, off, s[0:3], s32 offset:252
; GFX9-NEXT: buffer_load_ushort v10, off, s[0:3], s32 offset:248
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:244
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:240
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -195497,10 +195589,10 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:208
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:200
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32 offset:204
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:196
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -195508,66 +195600,66 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:192
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:184
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:188
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:180
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
+; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
+; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
+; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
+; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
+; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
+; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
+; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
+; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
+; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
+; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
+; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
+; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
+; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
+; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
+; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
+; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
+; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
; GFX9-NEXT: buffer_load_ushort v61, off, s[0:3], s32 offset:140
; GFX9-NEXT: buffer_load_ushort v36, off, s[0:3], s32 offset:136
; GFX9-NEXT: buffer_load_ushort v60, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:112
; GFX9-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:108
-; GFX9-NEXT: buffer_load_ushort v28, off, s[0:3], s32 offset:104
-; GFX9-NEXT: buffer_load_ushort v35, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_ushort v48, off, s[0:3], s32 offset:96
-; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:92
-; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
-; GFX9-NEXT: buffer_load_ushort v29, off, s[0:3], s32 offset:84
-; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:72
-; GFX9-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:68
-; GFX9-NEXT: buffer_load_ushort v30, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ushort v33, off, s[0:3], s32 offset:60
-; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ushort v21, off, s[0:3], s32 offset:52
-; GFX9-NEXT: buffer_load_ushort v26, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ushort v31, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_ushort v15, off, s[0:3], s32 offset:40
-; GFX9-NEXT: buffer_load_ushort v18, off, s[0:3], s32 offset:36
-; GFX9-NEXT: buffer_load_ushort v22, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_ushort v27, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32 offset:24
-; GFX9-NEXT: buffer_load_ushort v16, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v19, off, s[0:3], s32 offset:16
-; GFX9-NEXT: buffer_load_ushort v23, off, s[0:3], s32 offset:12
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
; GFX9-NEXT: s_waitcnt vmcnt(33)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:172
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v32, off, s[0:3], s32 offset:176
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:168
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:164
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:160
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:156
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:152
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:148
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:144
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
@@ -195592,7 +195684,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(27)
+; GFX9-NEXT: s_waitcnt vmcnt(26)
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
@@ -195618,7 +195710,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(49)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; GFX9-NEXT: s_cbranch_scc0 .LBB97_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
@@ -195730,8 +195821,10 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v8, v35, v34, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: v_perm_b32 v10, v58, v43, s4
+; GFX9-NEXT: s_waitcnt vmcnt(56)
; GFX9-NEXT: v_mov_b32_e32 v53, v59
; GFX9-NEXT: v_or_b32_sdwa v19, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_waitcnt vmcnt(55)
; GFX9-NEXT: v_perm_b32 v8, v41, v53, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
@@ -195860,6 +195953,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: v_mov_b32_e32 v63, v39
; GFX9-NEXT: v_mov_b32_e32 v38, v36
; GFX9-NEXT: v_mov_b32_e32 v44, v55
+; GFX9-NEXT: s_waitcnt vmcnt(51)
; GFX9-NEXT: v_mov_b32_e32 v50, v59
; GFX9-NEXT: v_mov_b32_e32 v35, v60
; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
@@ -196297,6 +196391,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: ; meta instruction
; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:320
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
@@ -196328,8 +196423,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
@@ -196361,8 +196456,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
@@ -196371,7 +196466,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
@@ -196411,7 +196505,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
@@ -196601,15 +196695,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(55)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
@@ -196618,12 +196715,14 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
@@ -196631,15 +196730,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
@@ -196648,12 +196750,14 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
@@ -196661,15 +196765,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
@@ -196678,6 +196785,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
@@ -196818,6 +196926,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s32 offset:320 ; 4-byte Folded Spill
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:312
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:308
; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:304
@@ -196849,8 +196958,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:200
; GFX11-FAKE16-NEXT: scratch_load_u16 v130, off, s32 offset:196
; GFX11-FAKE16-NEXT: scratch_load_u16 v119, off, s32 offset:192
-; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_u16 v134, off, s32 offset:188
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:184
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:180
; GFX11-FAKE16-NEXT: scratch_load_u16 v129, off, s32 offset:176
@@ -196882,8 +196991,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
-; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
@@ -196892,7 +197001,6 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
@@ -196932,7 +197040,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(62)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
@@ -197160,15 +197268,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v150, 0x300, v4
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v151, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(59)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v162
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v149
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v135, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(57)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v148
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v145, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v133, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v144, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(43)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v132
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v135, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v146
@@ -197188,11 +197299,13 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v115
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v115, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(39)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v118
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v113
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v99, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(37)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v112
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v117, 0xc0c0004
@@ -197200,12 +197313,14 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v97, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v100, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v99, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v102
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v114, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v87
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v84, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v96
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v103, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v2
@@ -197218,17 +197333,21 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v71
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v71, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
@@ -200499,8 +200618,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr34
; GFX9-NEXT: ; kill: killed $vgpr34
@@ -200676,7 +200795,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(13)
+; GFX9-NEXT: s_waitcnt vmcnt(14)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v33
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -200692,6 +200811,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v15
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v14
+; GFX9-NEXT: s_waitcnt vmcnt(17)
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 24, v32
; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v14
@@ -200863,6 +200983,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB98_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
+; GFX9-NEXT: s_waitcnt vmcnt(13)
; GFX9-NEXT: v_pk_add_u16 v32, v32, 3 op_sel_hi:[1,0]
; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_pk_add_u16 v31, v31, 3 op_sel_hi:[1,0]
@@ -205971,9 +206092,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v28, v51, v52, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v26, v15, v51, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -205982,9 +206103,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v20, v20, v15, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
; GFX9-NEXT: buffer_store_dword v39, v0, s[0:3], 0 offen
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
@@ -211044,13 +211165,13 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_mov_b32_e32 v8, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
@@ -211096,10 +211217,9 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
@@ -211117,8 +211237,9 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_lshr_b64 v[0:1], v[42:43], 16
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_mov_b32_e32 v21, v10
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
@@ -211356,15 +211477,15 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_mov_b32_e32 v17, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_mov_b32_e32 v60, v0
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
@@ -211908,71 +212029,70 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; SI-NEXT: .LBB101_5: ; %end
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v4
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v5
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v26
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v22
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v58
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: v_readlane_b32 s30, v63, 3
; SI-NEXT: v_readlane_b32 s31, v63, 4
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: v_and_b32_e32 v9, 0xffff, v12
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v21
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; SI-NEXT: v_or_b32_e32 v0, v0, v2
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v14
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v20
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v57
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v10
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
@@ -211980,10 +212100,10 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
@@ -211991,15 +212111,15 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_or_b32_e32 v3, v3, v4
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v35
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v10
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v6, v6, v10
@@ -212020,15 +212140,15 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_or_b32_e32 v9, v9, v10
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v39
; SI-NEXT: v_or_b32_e32 v11, v12, v11
; SI-NEXT: s_waitcnt vmcnt(1)
@@ -212041,45 +212161,45 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v25
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v40
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v46
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v16, v16, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v17, v18, v17
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v18, v18, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v19, v19, v20
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v20, v20, v22
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -212088,8 +212208,8 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v24
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v49
; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v25
; SI-NEXT: v_or_b32_e32 v23, v23, v24
@@ -212097,33 +212217,33 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v34
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
; SI-NEXT: v_or_b32_e32 v24, v24, v25
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v47
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v35
; SI-NEXT: v_or_b32_e32 v25, v25, v26
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v27
; SI-NEXT: v_or_b32_e32 v26, v26, v29
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v55
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v35
; SI-NEXT: v_or_b32_e32 v27, v27, v29
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v28
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v53
@@ -222292,11 +222412,11 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
@@ -222323,11 +222443,11 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v25
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -222389,16 +222509,16 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
; SI-NEXT: v_or_b32_e32 v24, v24, v25
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v29
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
@@ -222413,8 +222533,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v27
; SI-NEXT: v_or_b32_e32 v26, v26, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v56
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
@@ -222447,8 +222567,8 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v29, v29, v30
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v38
; SI-NEXT: v_or_b32_e32 v30, v30, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
; SI-NEXT: v_or_b32_e32 v31, v31, v32
@@ -228641,8 +228761,8 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v45
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 24cebc1097205..db7bc2a31c1b2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -34048,8 +34048,8 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
@@ -34062,7 +34062,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
@@ -34084,6 +34084,7 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB98_3
@@ -39613,8 +39614,8 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
@@ -39627,7 +39628,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
@@ -39649,6 +39650,7 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB106_3
@@ -44048,8 +44050,8 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
@@ -44062,7 +44064,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
@@ -44084,6 +44086,7 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB110_3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index c0fcebcf00814..8a0c2b8b1d884 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -4195,14 +4195,14 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_mov_b32_e32 v33, v4
; SI-NEXT: v_mov_b32_e32 v32, v2
; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4
@@ -4222,17 +4222,17 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v23, 24, v27
; SI-NEXT: v_lshlrev_b32_e32 v21, 8, v29
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v0
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v8
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v44
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v13, 8, v6
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v8
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v44
+; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v4
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -4482,8 +4482,8 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v48, v0
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:24
@@ -4501,7 +4501,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v37, v3
; VI-NEXT: v_mov_b32_e32 v38, v2
; VI-NEXT: v_mov_b32_e32 v39, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -4536,6 +4536,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v55, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -4663,6 +4664,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; VI-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v8, 3, v30
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_lshlrev_b16_e32 v10, 8, v55
; VI-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -4700,8 +4702,8 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v48, v0
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:32
; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:24
@@ -4719,7 +4721,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v37, v3
; GFX9-NEXT: v_mov_b32_e32 v38, v2
; GFX9-NEXT: v_mov_b32_e32 v39, v1
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -4749,6 +4751,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_perm_b32 v8, v30, v55, s6
; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -4869,6 +4872,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v8, 3, v30
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v9, 8, v55
; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -4904,11 +4908,11 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
+; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
@@ -4926,7 +4930,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_3
@@ -5124,11 +5128,11 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v48, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:8
@@ -5138,7 +5142,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v34, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v36, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v38, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -5308,6 +5312,7 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v27
; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v11, v7
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v12, v53, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -11101,14 +11106,14 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_mov_b32_e32 v33, v4
; SI-NEXT: v_mov_b32_e32 v32, v2
; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:4
@@ -11128,17 +11133,17 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v23, 24, v27
; SI-NEXT: v_lshlrev_b32_e32 v21, 8, v29
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v19, 24, v0
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v8
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v44
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v13, 8, v6
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 8, v8
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v15, 24, v44
+; SI-NEXT: v_lshlrev_b32_e32 v11, 24, v4
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -11388,8 +11393,8 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v48, v0
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:24
@@ -11407,7 +11412,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v37, v3
; VI-NEXT: v_mov_b32_e32 v38, v2
; VI-NEXT: v_mov_b32_e32 v39, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -11442,6 +11447,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v55, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -11569,6 +11575,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; VI-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v8, 3, v30
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_lshlrev_b16_e32 v10, 8, v55
; VI-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -11606,8 +11613,8 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v48, v0
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v49, off, s[0:3], s32 offset:32
; GFX9-NEXT: buffer_load_ushort v50, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v51, off, s[0:3], s32 offset:24
@@ -11625,7 +11632,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v37, v3
; GFX9-NEXT: v_mov_b32_e32 v38, v2
; GFX9-NEXT: v_mov_b32_e32 v39, v1
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -11655,6 +11662,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_perm_b32 v8, v30, v55, s6
; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -11775,6 +11783,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v8, 3, v30
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v9, 8, v55
; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -11810,11 +11819,11 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
+; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
@@ -11832,7 +11841,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_3
@@ -12030,11 +12039,11 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v48, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:8
@@ -12044,7 +12053,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v34, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v36, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v38, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -12214,6 +12223,7 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v27
; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v11, v7
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v12, v53, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -17557,15 +17567,15 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:24
; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:24
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:4
; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v5
; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v7
@@ -17598,23 +17608,23 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: s_waitcnt vmcnt(9) expcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT: s_waitcnt vmcnt(7) expcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(8) expcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v35
+; SI-NEXT: s_waitcnt vmcnt(7) expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v55
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v54
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v40
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 8, v41
; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: s_waitcnt vmcnt(5) expcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v55
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v40
; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v57, 8, v41
+; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -17639,6 +17649,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v10, v38, v5
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v15, 0xff, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
@@ -17652,6 +17663,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v9, 0xff, v30
; SI-NEXT: v_or_b32_e32 v18, v56, v15
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v15, 0xff, v43
; SI-NEXT: v_or_b32_e32 v0, v0, v48
; SI-NEXT: v_or_b32_e32 v11, v52, v7
@@ -17927,8 +17939,8 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v31, v0
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:24
@@ -17952,7 +17964,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v36, v3
; VI-NEXT: v_mov_b32_e32 v37, v2
; VI-NEXT: v_mov_b32_e32 v33, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -17987,6 +17999,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v55, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -18169,14 +18182,14 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v34, v2
; GFX9-NEXT: v_mov_b32_e32 v31, v1
; GFX9-NEXT: v_mov_b32_e32 v38, v0
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:24
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:4
; GFX9-NEXT: v_mov_b32_e32 v39, v15
@@ -18191,14 +18204,14 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v48, v6
; GFX9-NEXT: v_mov_b32_e32 v37, v5
; GFX9-NEXT: v_mov_b32_e32 v51, v4
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_lshlrev_b16_e32 v47, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v46, 8, v2
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b16_e32 v45, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshlrev_b16_e32 v47, 8, v1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -18228,6 +18241,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_perm_b32 v6, v7, v6, s7
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_perm_b32 v8, v30, v55, s6
; GFX9-NEXT: v_perm_b32 v7, v8, v7, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -18281,6 +18295,7 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB50_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_add_u16_e32 v0, 3, v44
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v43
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -18387,10 +18402,10 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
@@ -18419,12 +18434,12 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
@@ -18432,10 +18447,10 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
@@ -18634,10 +18649,10 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, v4 :: v_dual_mov_b32 v35, v2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, v1 :: v_dual_mov_b32 v37, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
-; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_b32 v1, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:8
; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v70, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:20
@@ -18650,13 +18665,13 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, v11 :: v_dual_mov_b32 v31, v8
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, v7 :: v_dual_mov_b32 v39, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v66, 8, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v65, 8, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v64, 8, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v66, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -24008,15 +24023,15 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:36
+; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32
+; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:24
; SI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:24
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:20
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:16
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:12
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:4
; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v5
; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v7
@@ -24049,23 +24064,23 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr7
; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr9
-; SI-NEXT: s_waitcnt vmcnt(9) expcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v53
-; SI-NEXT: s_waitcnt vmcnt(7) expcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(8) expcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v35
+; SI-NEXT: s_waitcnt vmcnt(7) expcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v55
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_lshlrev_b32_e32 v56, 24, v54
+; SI-NEXT: s_waitcnt vmcnt(5) expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v40
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v57, 8, v41
; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: ; implicit-def: $vgpr35
-; SI-NEXT: s_waitcnt vmcnt(5) expcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v58, 8, v55
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v40
; SI-NEXT: ; implicit-def: $vgpr40
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v57, 8, v41
+; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -24090,6 +24105,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v10, v38, v5
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_and_b32_e32 v15, 0xff, v50
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_and_b32_e32 v1, 0xff, v4
@@ -24103,6 +24119,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; SI-NEXT: v_and_b32_e32 v9, 0xff, v30
; SI-NEXT: v_or_b32_e32 v18, v56, v15
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v15, 0xff, v43
; SI-NEXT: v_or_b32_e32 v0, v0, v48
; SI-NEXT: v_or_b32_e32 v11, v52, v7
@@ -24378,8 +24395,8 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v31, v0
-; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v46, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v47, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:24
@@ -24403,7 +24420,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v36, v3
; VI-NEXT: v_mov_b32_e32 v37, v2
; VI-NEXT: v_mov_b32_e32 v33, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -24438,6 +24455,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v55, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -24620,14 +24638,14 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v34, v2
; GFX9-NEXT: v_mov_b32_e32 v31, v1
; GFX9-NEXT: v_mov_b32_e32 v38, v0
-; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32 offset:24
; GFX9-NEXT: buffer_load_ushort v44, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_ushort v2, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_ushort v3, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_ushort v42, off, s[0:3], s32 offset:4
; GFX9-NEXT: v_mov_b32_e32 v39, v15
@@ -24642,14 +24660,14 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v48, v6
; GFX9-NEXT: v_mov_b32_e32 v37, v5
; GFX9-NEXT: v_mov_b32_e32 v51, v4
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_lshlrev_b16_e32 v47, 8, v1
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v46, 8, v2
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b16_e32 v45, 8, v3
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshlrev_b16_e32 v47, 8, v1
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -24679,6 +24697,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_perm_b32 v6, v7, v6, s7
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_perm_b32 v8, v30, v55, s6
; GFX9-NEXT: v_perm_b32 v7, v8, v7, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -24732,6 +24751,7 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB62_4
; GFX9-NEXT: ; %bb.3: ; %cmp.true
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_add_u16_e32 v0, 3, v44
; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v43
; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
@@ -24838,10 +24858,10 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
@@ -24870,12 +24890,12 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
@@ -24883,10 +24903,10 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_3
@@ -25085,10 +25105,10 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v52, v4 :: v_dual_mov_b32 v35, v2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v48, v1 :: v_dual_mov_b32 v37, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
-; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_b32 v1, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:8
; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v70, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:20
@@ -25101,13 +25121,13 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v38, v11 :: v_dual_mov_b32 v31, v8
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v36, v7 :: v_dual_mov_b32 v39, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v66, 8, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v65, 8, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v64, 8, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(6)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v66, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -28578,13 +28598,13 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_mov_b32_e32 v33, v4
; SI-NEXT: v_mov_b32_e32 v32, v2
; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:4
@@ -28607,17 +28627,17 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v51, 24, v27
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v29
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v0
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v8
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v10
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v6
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v8
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v4
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -28877,8 +28897,8 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v54, v0
-; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:24
@@ -28902,7 +28922,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: v_mov_b32_e32 v53, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -28937,6 +28957,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v45, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -29064,6 +29085,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; VI-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v8, 3, v30
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_lshlrev_b16_e32 v10, 8, v45
; VI-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -29113,8 +29135,8 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v54, v0
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:32
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:24
@@ -29138,7 +29160,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: v_mov_b32_e32 v53, v1
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -29168,6 +29190,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_perm_b32 v8, v30, v45, s6
; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -29288,6 +29311,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v8, 3, v30
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v9, 8, v45
; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -29329,11 +29353,11 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
+; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
@@ -29358,7 +29382,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB72_3
@@ -29556,11 +29580,11 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:8
@@ -29573,7 +29597,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -29743,6 +29767,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v27
; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v11, v7
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v12, v67, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
@@ -32443,13 +32468,13 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_mov_b32_e32 v33, v4
; SI-NEXT: v_mov_b32_e32 v32, v2
; SI-NEXT: v_mov_b32_e32 v31, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:4
@@ -32472,17 +32497,17 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v51, 24, v27
; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v29
; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v0
-; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v0
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v4
+; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v8
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v10
; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v19, 8, v6
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v23, 8, v8
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v21, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v17, 24, v4
; SI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
@@ -32742,8 +32767,8 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v54, v0
-; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; VI-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:32
; VI-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:28
; VI-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:24
@@ -32767,7 +32792,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: v_mov_b32_e32 v53, v1
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -32802,6 +32827,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: v_perm_b32 v6, v24, v25, s6
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v8, v30, v45, s6
; VI-NEXT: v_or_b32_e32 v6, v6, v7
; VI-NEXT: v_perm_b32 v7, v28, v29, s6
@@ -32929,6 +32955,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; VI-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v8, 3, v30
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_lshlrev_b16_e32 v10, 8, v45
; VI-NEXT: v_or_b32_sdwa v8, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; VI-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -32978,8 +33005,8 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v54, v0
-; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ushort v45, off, s[0:3], s32
; GFX9-NEXT: buffer_load_ushort v55, off, s[0:3], s32 offset:32
; GFX9-NEXT: buffer_load_ushort v40, off, s[0:3], s32 offset:28
; GFX9-NEXT: buffer_load_ushort v41, off, s[0:3], s32 offset:24
@@ -33003,7 +33030,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: v_mov_b32_e32 v53, v1
-; GFX9-NEXT: s_waitcnt vmcnt(8)
+; GFX9-NEXT: s_waitcnt vmcnt(9)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -33033,6 +33060,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v26, v27, s6
; GFX9-NEXT: v_lshl_or_b32 v6, v7, 16, v6
; GFX9-NEXT: v_perm_b32 v7, v28, v29, s6
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_perm_b32 v8, v30, v45, s6
; GFX9-NEXT: v_lshl_or_b32 v7, v8, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -33153,6 +33181,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshlrev_b16_e32 v8, 8, v29
; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v8, 3, v30
+; GFX9-NEXT: s_waitcnt vmcnt(8)
; GFX9-NEXT: v_lshlrev_b16_e32 v9, 8, v45
; GFX9-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v7, 0x300, v7
@@ -33194,11 +33223,11 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x9
+; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
@@ -33223,7 +33252,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3
@@ -33421,11 +33450,11 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x9
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:8
@@ -33438,7 +33467,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -33608,6 +33637,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v27
; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v11, v7
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v12, v8
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v12, v67, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 7a26f3497f492..9f696b23a4136 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -11693,17 +11693,17 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v44
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v50, 24, v58
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v47
@@ -12212,9 +12212,9 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v53, v1
; VI-NEXT: v_mov_b32_e32 v54, v0
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -12252,11 +12252,11 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: s_waitcnt vmcnt(14)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -12684,9 +12684,9 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v53, v1
; GFX9-NEXT: v_mov_b32_e32 v54, v0
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -12724,13 +12724,13 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: s_waitcnt vmcnt(24)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -13115,6 +13115,7 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
@@ -13143,7 +13144,6 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
@@ -13169,7 +13169,7 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3
@@ -13467,6 +13467,7 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:120
@@ -13495,7 +13496,6 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v112, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v114, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v118, off, s32 offset:8
@@ -13509,7 +13509,7 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -13875,20 +13875,20 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s47, v13
; SI-NEXT: v_readfirstlane_b32 s56, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:28
@@ -13911,22 +13911,21 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v14
-; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
; SI-NEXT: v_lshlrev_b32_e32 v30, 8, v22
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
; SI-NEXT: s_cbranch_scc0 .LBB27_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
@@ -14005,6 +14004,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s90, s90, s91
; SI-NEXT: s_and_b32 s91, s42, 0xff
; SI-NEXT: s_lshl_b32 s92, s41, 8
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_and_b32_e32 v1, 0xff, v20
; SI-NEXT: s_or_b32 s91, s91, s92
; SI-NEXT: s_and_b32 s92, s40, 0xff
@@ -14119,6 +14119,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s26, 0xff
; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s27, 24
@@ -14928,6 +14929,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:48
@@ -14936,7 +14938,6 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
@@ -14976,12 +14977,13 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
@@ -15217,6 +15219,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:48
@@ -15225,7 +15228,6 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -15265,12 +15267,13 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
@@ -26888,17 +26891,17 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v44
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v50, 24, v58
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v47
@@ -27407,9 +27410,9 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v53, v1
; VI-NEXT: v_mov_b32_e32 v54, v0
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -27447,11 +27450,11 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: s_waitcnt vmcnt(14)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -27879,9 +27882,9 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v53, v1
; GFX9-NEXT: v_mov_b32_e32 v54, v0
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -27919,13 +27922,13 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: s_waitcnt vmcnt(24)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -28310,6 +28313,7 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
@@ -28338,7 +28342,6 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
@@ -28364,7 +28367,7 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
@@ -28662,6 +28665,7 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:120
@@ -28690,7 +28694,6 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v112, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v114, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v118, off, s32 offset:8
@@ -28704,7 +28707,7 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -29070,20 +29073,20 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: v_readfirstlane_b32 s47, v13
; SI-NEXT: v_readfirstlane_b32 s56, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:28
@@ -29106,22 +29109,21 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v14
-; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
; SI-NEXT: v_lshlrev_b32_e32 v30, 8, v22
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
; SI-NEXT: s_cbranch_scc0 .LBB51_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
@@ -29200,6 +29202,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s90, s90, s91
; SI-NEXT: s_and_b32 s91, s42, 0xff
; SI-NEXT: s_lshl_b32 s92, s41, 8
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_and_b32_e32 v1, 0xff, v20
; SI-NEXT: s_or_b32 s91, s91, s92
; SI-NEXT: s_and_b32 s92, s40, 0xff
@@ -29314,6 +29317,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s26, 0xff
; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s27, 24
@@ -30123,6 +30127,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:48
@@ -30131,7 +30136,6 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
@@ -30171,12 +30175,13 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
@@ -30412,6 +30417,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:48
@@ -30420,7 +30426,6 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -30460,12 +30465,13 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
@@ -40961,17 +40967,17 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v44
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v50, 24, v58
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v47
@@ -41480,9 +41486,9 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v53, v1
; VI-NEXT: v_mov_b32_e32 v54, v0
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -41520,11 +41526,11 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: s_waitcnt vmcnt(14)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -41952,9 +41958,9 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v53, v1
; GFX9-NEXT: v_mov_b32_e32 v54, v0
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -41992,13 +41998,13 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: s_waitcnt vmcnt(24)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -42383,6 +42389,7 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
@@ -42411,7 +42418,6 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
@@ -42437,7 +42443,7 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3
@@ -42735,6 +42741,7 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:120
@@ -42763,7 +42770,6 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v112, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v114, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v118, off, s32 offset:8
@@ -42777,7 +42783,7 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -43143,20 +43149,20 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: v_readfirstlane_b32 s47, v13
; SI-NEXT: v_readfirstlane_b32 s56, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:28
@@ -43179,22 +43185,21 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v14
-; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
; SI-NEXT: v_lshlrev_b32_e32 v30, 8, v22
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
; SI-NEXT: s_cbranch_scc0 .LBB71_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
@@ -43273,6 +43278,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s90, s90, s91
; SI-NEXT: s_and_b32 s91, s42, 0xff
; SI-NEXT: s_lshl_b32 s92, s41, 8
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_and_b32_e32 v1, 0xff, v20
; SI-NEXT: s_or_b32 s91, s91, s92
; SI-NEXT: s_and_b32 s92, s40, 0xff
@@ -43387,6 +43393,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s26, 0xff
; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s27, 24
@@ -44196,6 +44203,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:48
@@ -44204,7 +44212,6 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
@@ -44244,12 +44251,13 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
@@ -44485,6 +44493,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:48
@@ -44493,7 +44502,6 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -44533,12 +44541,13 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
@@ -54452,17 +54461,17 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v44
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:28
; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:20
; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_lshlrev_b32_e32 v50, 24, v58
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v47
@@ -54971,9 +54980,9 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v53, v1
; VI-NEXT: v_mov_b32_e32 v54, v0
+; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; VI-NEXT: buffer_load_ushort v30, off, s[0:3], s32
-; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; VI-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; VI-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -55011,11 +55020,11 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; VI-NEXT: v_mov_b32_e32 v51, v3
; VI-NEXT: v_mov_b32_e32 v52, v2
; VI-NEXT: s_waitcnt vmcnt(14)
-; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_ushort v25, off, s[0:3], s32 offset:124
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -55443,9 +55452,9 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v53, v1
; GFX9-NEXT: v_mov_b32_e32 v54, v0
+; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; GFX9-NEXT: buffer_load_ushort v43, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:128
; GFX9-NEXT: buffer_load_ushort v57, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_ushort v58, off, s[0:3], s32 offset:84
; GFX9-NEXT: buffer_load_ushort v59, off, s[0:3], s32 offset:80
@@ -55483,13 +55492,13 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX9-NEXT: v_mov_b32_e32 v51, v3
; GFX9-NEXT: v_mov_b32_e32 v52, v2
; GFX9-NEXT: s_waitcnt vmcnt(24)
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_waitcnt vmcnt(22)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:120
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_ushort v56, off, s[0:3], s32 offset:124
+; GFX9-NEXT: s_waitcnt vmcnt(26)
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v1, off, s[0:3], s32 offset:116
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -55874,6 +55883,7 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:128
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:120
@@ -55902,7 +55912,6 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v128, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:8
@@ -55928,7 +55937,7 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v128
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3
@@ -56226,6 +56235,7 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v53, v1 :: v_dual_mov_b32 v54, v0
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:120
@@ -56254,7 +56264,6 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v103, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v112, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v114, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_b32 v0, off, s32 offset:132
; GFX11-FAKE16-NEXT: scratch_load_u16 v115, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v117, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v118, off, s32 offset:8
@@ -56268,7 +56277,7 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v48, v6
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v49, v5 :: v_dual_mov_b32 v50, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v3 :: v_dual_mov_b32 v52, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -56634,20 +56643,20 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: v_readfirstlane_b32 s47, v13
; SI-NEXT: v_readfirstlane_b32 s56, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:8
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:24
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:32
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
+; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:72
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:60
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:56
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48
; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:36
; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:28
@@ -56670,22 +56679,21 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
; SI-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v14
-; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
+; SI-NEXT: v_lshlrev_b32_e32 v33, 24, v13
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
; SI-NEXT: v_lshlrev_b32_e32 v30, 8, v22
; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 24, v24
+; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v15
; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v25
+; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
; SI-NEXT: s_waitcnt vmcnt(11)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
-; SI-NEXT: s_waitcnt vmcnt(9)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
-; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshlrev_b32_e32 v25, 24, v37
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshlrev_b32_e32 v27, 8, v27
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 8, v36
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 24, v35
; SI-NEXT: s_cbranch_scc0 .LBB87_2
; SI-NEXT: ; %bb.1: ; %cmp.false
; SI-NEXT: s_and_b32 s4, s6, 0xff
@@ -56764,6 +56772,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s90, s90, s91
; SI-NEXT: s_and_b32 s91, s42, 0xff
; SI-NEXT: s_lshl_b32 s92, s41, 8
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_and_b32_e32 v1, 0xff, v20
; SI-NEXT: s_or_b32 s91, s91, s92
; SI-NEXT: s_and_b32 s92, s40, 0xff
@@ -56878,6 +56887,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_and_b32 s18, s26, 0xff
; SI-NEXT: s_and_b32 s6, s6, 0xff
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_add_i32_e32 v1, vcc, 3, v20
; SI-NEXT: s_addk_i32 s16, 0x300
; SI-NEXT: s_lshl_b32 s17, s27, 24
@@ -57687,6 +57697,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:48
@@ -57695,7 +57706,6 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
@@ -57735,12 +57745,13 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
@@ -57976,6 +57987,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:48
@@ -57984,7 +57996,6 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -58024,12 +58035,13 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(9)
; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
@@ -68025,6 +68037,8 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32
@@ -68047,7 +68061,6 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -68099,18 +68112,20 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v60, 8, v4
; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v62, 8, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v63, 24, v16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v9
-; SI-NEXT: v_lshlrev_b32_e32 v63, 24, v16
-; SI-NEXT: v_lshlrev_b32_e32 v42, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v20
-; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v42, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v26
; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v28
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68
@@ -68118,6 +68133,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v10
; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v14
; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v18
; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v22
@@ -68127,9 +68143,6 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v17
; SI-NEXT: v_lshlrev_b32_e32 v31, 8, v32
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v33
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v35
; SI-NEXT: v_lshlrev_b32_e32 v51, 8, v36
@@ -68143,7 +68156,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
@@ -69691,29 +69704,29 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
@@ -69737,15 +69750,15 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
@@ -69762,18 +69775,20 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB98_3
@@ -69913,6 +69928,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
@@ -69922,6 +69938,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
@@ -70078,29 +70095,29 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v5 :: v_dual_mov_b32 v38, v0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v50, v1
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v102, off, s32 offset:100
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:92
; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v81, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:52
@@ -70114,31 +70131,31 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(32)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v84, 8, v7
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v65, 8, v8
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(14)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v80, 8, v9
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v112, 8, v10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(12)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v113, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v103, 8, v12
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -70280,13 +70297,16 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-FAKE16-NEXT: .LBB98_4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v99, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v100, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v87
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v102, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v86, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v81, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v23
@@ -70299,6 +70319,7 @@ define <32 x i16> @bitcast_v64i8_to_v32i16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v82, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v81, 0x300, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v64
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v85, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v14, 0x300, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -71888,6 +71909,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
@@ -71896,7 +71918,6 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
@@ -71936,7 +71957,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
@@ -72029,6 +72050,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -72153,6 +72175,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
@@ -72161,7 +72184,6 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
@@ -72201,7 +72223,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
@@ -72308,6 +72330,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -75568,9 +75591,9 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v53
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -75592,8 +75615,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_or_b32_e32 v5, v5, v6
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v54
; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
@@ -75605,8 +75628,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v18
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff, v29
@@ -75644,8 +75667,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v55
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; SI-NEXT: v_or_b32_e32 v15, v15, v16
@@ -80814,6 +80837,8 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32
@@ -80836,7 +80861,6 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:76
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:36
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -80888,18 +80912,20 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v60, 8, v4
; SI-NEXT: v_lshlrev_b32_e32 v46, 24, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v1
; SI-NEXT: v_lshlrev_b32_e32 v62, 8, v12
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; SI-NEXT: v_lshlrev_b32_e32 v63, 24, v16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v9
-; SI-NEXT: v_lshlrev_b32_e32 v63, 24, v16
-; SI-NEXT: v_lshlrev_b32_e32 v42, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v20
-; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v42, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v34, 8, v26
; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v28
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68
@@ -80907,6 +80933,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12
+; SI-NEXT: v_lshlrev_b32_e32 v57, 24, v10
; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v14
; SI-NEXT: v_lshlrev_b32_e32 v18, 8, v18
; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v22
@@ -80916,9 +80943,6 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v17
; SI-NEXT: v_lshlrev_b32_e32 v31, 8, v32
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:28
; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v33
; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v35
; SI-NEXT: v_lshlrev_b32_e32 v51, 8, v36
@@ -80932,7 +80956,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr22
; SI-NEXT: ; implicit-def: $vgpr17
; SI-NEXT: ; implicit-def: $vgpr33
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
@@ -82480,29 +82504,29 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
@@ -82526,15 +82550,15 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
@@ -82551,18 +82575,20 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB106_3
@@ -82702,6 +82728,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
@@ -82711,6 +82738,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
@@ -82867,29 +82895,29 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v5 :: v_dual_mov_b32 v38, v0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v50, v1
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v102, off, s32 offset:100
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:92
; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v81, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:52
@@ -82903,31 +82931,31 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(32)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v84, 8, v7
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v65, 8, v8
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(14)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v80, 8, v9
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v112, 8, v10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(12)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v113, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v103, 8, v12
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -83069,13 +83097,16 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-FAKE16-NEXT: .LBB106_4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v99, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v100, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v87
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v102, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v86, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v81, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v23
@@ -83088,6 +83119,7 @@ define <32 x half> @bitcast_v64i8_to_v32f16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v82, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v81, 0x300, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v64
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v85, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v14, 0x300, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -84677,6 +84709,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
@@ -84685,7 +84718,6 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
@@ -84725,7 +84757,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
@@ -84818,6 +84850,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -84942,6 +84975,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
@@ -84950,7 +84984,6 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
@@ -84990,7 +85023,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
@@ -85097,6 +85130,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -91528,6 +91562,8 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32
@@ -91548,7 +91584,6 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:100
; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:92
; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:28
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -91602,21 +91637,22 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: ; implicit-def: $vgpr35
; SI-NEXT: ; implicit-def: $vgpr19
; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:20
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v1
; SI-NEXT: v_lshlrev_b32_e32 v40, 24, v2
; SI-NEXT: v_lshlrev_b32_e32 v60, 8, v6
-; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v8
+; SI-NEXT: v_lshlrev_b32_e32 v54, 24, v1
; SI-NEXT: v_lshlrev_b32_e32 v59, 24, v9
-; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v14
-; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v47, 24, v8
; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v16
; SI-NEXT: v_lshlrev_b32_e32 v26, 24, v17
; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v18
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_lshlrev_b32_e32 v16, 24, v20
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v24
-; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: v_lshlrev_b32_e32 v22, 8, v14
; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v25
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
@@ -91626,17 +91662,13 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:40
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v24
; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v28
; SI-NEXT: ; implicit-def: $vgpr14
; SI-NEXT: ; implicit-def: $vgpr24
; SI-NEXT: ; implicit-def: $vgpr28
-; SI-NEXT: s_waitcnt vmcnt(13)
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:20
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8
@@ -93202,29 +93234,29 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:120
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:112
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:104
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:96
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:80
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v100, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:64
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:52
@@ -93248,15 +93280,15 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v22.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v20.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v18.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(24)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v17.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v15.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v14.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l
@@ -93273,18 +93305,20 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v52.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v53.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v54.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v55.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v64.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v49.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v48.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.h
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v100
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB110_3
@@ -93424,6 +93458,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v33.h, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
@@ -93433,6 +93468,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.h, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v21.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v0.l
@@ -93589,29 +93625,29 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v39, v5 :: v_dual_mov_b32 v38, v0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v51, v4 :: v_dual_mov_b32 v50, v1
; GFX11-FAKE16-NEXT: s_clause 0x1f
+; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
+; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
+; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
+; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
+; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
+; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v0, off, s32 offset:128
; GFX11-FAKE16-NEXT: scratch_load_u16 v100, off, s32 offset:124
; GFX11-FAKE16-NEXT: scratch_load_u16 v87, off, s32 offset:120
; GFX11-FAKE16-NEXT: scratch_load_u16 v99, off, s32 offset:116
-; GFX11-FAKE16-NEXT: scratch_load_u16 v1, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v2, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v102, off, s32 offset:100
-; GFX11-FAKE16-NEXT: scratch_load_u16 v3, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:92
; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v81, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v4, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v5, off, s32 offset:72
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_b32 v6, off, s32 offset:132
-; GFX11-FAKE16-NEXT: scratch_load_u16 v7, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v8, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v9, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v10, off, s32 offset:40
-; GFX11-FAKE16-NEXT: scratch_load_u16 v11, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v12, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v97, off, s32 offset:52
@@ -93625,31 +93661,31 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(33)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(19)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(17)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(32)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v84, 8, v7
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(31)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v65, 8, v8
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(14)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(30)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v80, 8, v9
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(29)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v112, 8, v10
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(12)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(28)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v113, 8, v11
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(27)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v103, 8, v12
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(26)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v116, 8, v5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v114, 8, v4
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(24)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v115, 8, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(23)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v118, 8, v2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(22)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v117, 8, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(21)
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v119, 8, v0
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -93791,13 +93827,16 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-FAKE16-NEXT: .LBB110_4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(18)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v99, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v100, 3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v87
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(16)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v102, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v4, v86, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v81, 3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v23
@@ -93810,6 +93849,7 @@ define <32 x bfloat> @bitcast_v64i8_to_v32bf16(<64 x i8> %a, i32 %b) #0 {
; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v82, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v81, 0x300, v1
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v64
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u16 v5, v85, 3
; GFX11-FAKE16-NEXT: v_add_nc_u16 v14, 0x300, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -95368,6 +95408,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
@@ -95376,7 +95417,6 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
@@ -95416,7 +95456,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
@@ -95509,6 +95549,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
@@ -95633,6 +95674,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
+; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
@@ -95641,7 +95683,6 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
-; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
@@ -95681,7 +95722,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(15)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
@@ -95788,6 +95829,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(11)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index deced731f537b..72add1e3d202b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -53391,8 +53391,8 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v51
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_or_b32_e32 v24, v24, v26
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v44
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index abb378ef828dd..8f74e99aae097 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -7745,22 +7745,22 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GCN-NEXT: buffer_load_ushort v21, v[1:2], s[4:7], 0 addr64 offset:38
; GCN-NEXT: buffer_load_ushort v22, v[1:2], s[4:7], 0 addr64 offset:36
; GCN-NEXT: buffer_load_ushort v23, v[1:2], s[4:7], 0 addr64 offset:34
-; GCN-NEXT: buffer_load_ushort v24, v[1:2], s[4:7], 0 addr64 offset:32
; GCN-NEXT: buffer_load_ushort v25, v[1:2], s[4:7], 0 addr64
+; GCN-NEXT: buffer_load_ushort v24, v[1:2], s[4:7], 0 addr64 offset:32
; GCN-NEXT: buffer_load_ushort v26, v[1:2], s[4:7], 0 addr64 offset:2
-; GCN-NEXT: buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64 offset:16
-; GCN-NEXT: buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64 offset:18
-; GCN-NEXT: buffer_load_ushort v5, v[1:2], s[4:7], 0 addr64 offset:20
-; GCN-NEXT: buffer_load_ushort v6, v[1:2], s[4:7], 0 addr64 offset:22
-; GCN-NEXT: buffer_load_ushort v27, v[1:2], s[4:7], 0 addr64 offset:24
; GCN-NEXT: buffer_load_ushort v28, v[1:2], s[4:7], 0 addr64 offset:30
-; GCN-NEXT: buffer_load_ushort v29, v[1:2], s[4:7], 0 addr64 offset:26
-; GCN-NEXT: buffer_load_ushort v30, v[1:2], s[4:7], 0 addr64 offset:28
; GCN-NEXT: buffer_load_ushort v31, v[1:2], s[4:7], 0 addr64 offset:4
+; GCN-NEXT: buffer_load_ushort v30, v[1:2], s[4:7], 0 addr64 offset:28
; GCN-NEXT: buffer_load_ushort v32, v[1:2], s[4:7], 0 addr64 offset:6
+; GCN-NEXT: buffer_load_ushort v29, v[1:2], s[4:7], 0 addr64 offset:26
; GCN-NEXT: buffer_load_ushort v33, v[1:2], s[4:7], 0 addr64 offset:8
+; GCN-NEXT: buffer_load_ushort v27, v[1:2], s[4:7], 0 addr64 offset:24
; GCN-NEXT: buffer_load_ushort v34, v[1:2], s[4:7], 0 addr64 offset:10
+; GCN-NEXT: buffer_load_ushort v6, v[1:2], s[4:7], 0 addr64 offset:22
; GCN-NEXT: buffer_load_ushort v7, v[1:2], s[4:7], 0 addr64 offset:12
+; GCN-NEXT: buffer_load_ushort v5, v[1:2], s[4:7], 0 addr64 offset:20
+; GCN-NEXT: buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64 offset:16
+; GCN-NEXT: buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64 offset:18
; GCN-NEXT: buffer_load_ushort v1, v[1:2], s[4:7], 0 addr64 offset:14
; GCN-NEXT: s_waitcnt vmcnt(14)
; GCN-NEXT: v_lshlrev_b32_e32 v2, 16, v8
@@ -7934,22 +7934,22 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX7-NEXT: buffer_load_ushort v21, v[1:2], s[4:7], 0 addr64 offset:38
; GFX7-NEXT: buffer_load_ushort v22, v[1:2], s[4:7], 0 addr64 offset:36
; GFX7-NEXT: buffer_load_ushort v23, v[1:2], s[4:7], 0 addr64 offset:34
-; GFX7-NEXT: buffer_load_ushort v24, v[1:2], s[4:7], 0 addr64 offset:32
; GFX7-NEXT: buffer_load_ushort v25, v[1:2], s[4:7], 0 addr64
+; GFX7-NEXT: buffer_load_ushort v24, v[1:2], s[4:7], 0 addr64 offset:32
; GFX7-NEXT: buffer_load_ushort v26, v[1:2], s[4:7], 0 addr64 offset:2
-; GFX7-NEXT: buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64 offset:18
-; GFX7-NEXT: buffer_load_ushort v5, v[1:2], s[4:7], 0 addr64 offset:20
-; GFX7-NEXT: buffer_load_ushort v6, v[1:2], s[4:7], 0 addr64 offset:22
-; GFX7-NEXT: buffer_load_ushort v27, v[1:2], s[4:7], 0 addr64 offset:24
; GFX7-NEXT: buffer_load_ushort v28, v[1:2], s[4:7], 0 addr64 offset:30
-; GFX7-NEXT: buffer_load_ushort v29, v[1:2], s[4:7], 0 addr64 offset:26
-; GFX7-NEXT: buffer_load_ushort v30, v[1:2], s[4:7], 0 addr64 offset:28
; GFX7-NEXT: buffer_load_ushort v31, v[1:2], s[4:7], 0 addr64 offset:4
+; GFX7-NEXT: buffer_load_ushort v30, v[1:2], s[4:7], 0 addr64 offset:28
; GFX7-NEXT: buffer_load_ushort v32, v[1:2], s[4:7], 0 addr64 offset:6
+; GFX7-NEXT: buffer_load_ushort v29, v[1:2], s[4:7], 0 addr64 offset:26
; GFX7-NEXT: buffer_load_ushort v33, v[1:2], s[4:7], 0 addr64 offset:8
+; GFX7-NEXT: buffer_load_ushort v27, v[1:2], s[4:7], 0 addr64 offset:24
; GFX7-NEXT: buffer_load_ushort v34, v[1:2], s[4:7], 0 addr64 offset:10
+; GFX7-NEXT: buffer_load_ushort v6, v[1:2], s[4:7], 0 addr64 offset:22
; GFX7-NEXT: buffer_load_ushort v7, v[1:2], s[4:7], 0 addr64 offset:12
+; GFX7-NEXT: buffer_load_ushort v5, v[1:2], s[4:7], 0 addr64 offset:20
+; GFX7-NEXT: buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64 offset:16
+; GFX7-NEXT: buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64 offset:18
; GFX7-NEXT: buffer_load_ushort v1, v[1:2], s[4:7], 0 addr64 offset:14
; GFX7-NEXT: s_waitcnt vmcnt(14)
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v8
@@ -8171,19 +8171,19 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX8-NEXT: flat_load_ushort v8, v[7:8]
; GFX8-NEXT: v_add_u32_e32 v1, vcc, 30, v1
; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GFX8-NEXT: flat_load_ushort v12, v[1:2]
; GFX8-NEXT: flat_load_ushort v34, v[3:4]
+; GFX8-NEXT: flat_load_ushort v12, v[1:2]
; GFX8-NEXT: flat_load_ushort v41, v[5:6]
; GFX8-NEXT: flat_load_ushort v42, v[9:10]
+; GFX8-NEXT: flat_load_ushort v10, v[29:30]
; GFX8-NEXT: flat_load_ushort v14, v[13:14]
; GFX8-NEXT: flat_load_ushort v16, v[15:16]
; GFX8-NEXT: flat_load_ushort v1, v[19:20]
-; GFX8-NEXT: flat_load_ushort v10, v[29:30]
+; GFX8-NEXT: flat_load_ushort v13, v[27:28]
+; GFX8-NEXT: flat_load_ushort v15, v[25:26]
+; GFX8-NEXT: flat_load_ushort v19, v[23:24]
; GFX8-NEXT: flat_load_ushort v18, v[17:18]
; GFX8-NEXT: flat_load_ushort v17, v[21:22]
-; GFX8-NEXT: flat_load_ushort v19, v[23:24]
-; GFX8-NEXT: flat_load_ushort v15, v[25:26]
-; GFX8-NEXT: flat_load_ushort v13, v[27:28]
; GFX8-NEXT: s_waitcnt vmcnt(14)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v39
; GFX8-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
@@ -8338,22 +8338,22 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX900-NEXT: global_load_ushort v22, v[1:2], off offset:38
; GFX900-NEXT: global_load_ushort v23, v[1:2], off offset:36
; GFX900-NEXT: global_load_ushort v24, v[1:2], off offset:34
-; GFX900-NEXT: global_load_ushort v25, v[1:2], off offset:32
; GFX900-NEXT: global_load_ushort v26, v[1:2], off
+; GFX900-NEXT: global_load_ushort v25, v[1:2], off offset:32
; GFX900-NEXT: global_load_ushort v27, v[1:2], off offset:2
-; GFX900-NEXT: global_load_ushort v3, v[1:2], off offset:16
-; GFX900-NEXT: global_load_ushort v4, v[1:2], off offset:18
-; GFX900-NEXT: global_load_ushort v5, v[1:2], off offset:20
-; GFX900-NEXT: global_load_ushort v6, v[1:2], off offset:22
-; GFX900-NEXT: global_load_ushort v8, v[1:2], off offset:24
; GFX900-NEXT: global_load_ushort v28, v[1:2], off offset:30
-; GFX900-NEXT: global_load_ushort v29, v[1:2], off offset:26
-; GFX900-NEXT: global_load_ushort v30, v[1:2], off offset:28
; GFX900-NEXT: global_load_ushort v31, v[1:2], off offset:4
+; GFX900-NEXT: global_load_ushort v30, v[1:2], off offset:28
; GFX900-NEXT: global_load_ushort v32, v[1:2], off offset:6
+; GFX900-NEXT: global_load_ushort v29, v[1:2], off offset:26
; GFX900-NEXT: global_load_ushort v33, v[1:2], off offset:8
+; GFX900-NEXT: global_load_ushort v8, v[1:2], off offset:24
; GFX900-NEXT: global_load_ushort v34, v[1:2], off offset:10
+; GFX900-NEXT: global_load_ushort v6, v[1:2], off offset:22
; GFX900-NEXT: global_load_ushort v7, v[1:2], off offset:12
+; GFX900-NEXT: global_load_ushort v5, v[1:2], off offset:20
+; GFX900-NEXT: global_load_ushort v3, v[1:2], off offset:16
+; GFX900-NEXT: global_load_ushort v4, v[1:2], off offset:18
; GFX900-NEXT: s_nop 0
; GFX900-NEXT: global_load_ushort v1, v[1:2], off offset:14
; GFX900-NEXT: s_waitcnt vmcnt(31)
@@ -8426,21 +8426,22 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX900-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:148
; GFX900-NEXT: v_cvt_f64_f32_e32 v[9:10], v2
; GFX900-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:144
-; GFX900-NEXT: s_waitcnt vmcnt(44)
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v11, 16, v25
; GFX900-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:140
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:136
; GFX900-NEXT: v_cvt_f64_f32_e32 v[9:10], v11
-; GFX900-NEXT: s_waitcnt vmcnt(38)
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v13, 16, v28
; GFX900-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:132
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:128
; GFX900-NEXT: v_cvt_f64_f32_e32 v[9:10], v13
-; GFX900-NEXT: s_waitcnt vmcnt(38)
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v15, 16, v30
; GFX900-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:124
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:120
; GFX900-NEXT: v_cvt_f64_f32_e32 v[9:10], v15
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v17, 16, v29
; GFX900-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:116
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:112
@@ -8449,29 +8450,28 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX900-NEXT: v_cvt_f64_f32_e32 v[11:12], v2
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v27
; GFX900-NEXT: v_cvt_f64_f32_e32 v[13:14], v2
-; GFX900-NEXT: s_waitcnt vmcnt(41)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v31
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX900-NEXT: v_cvt_f64_f32_e32 v[15:16], v2
-; GFX900-NEXT: s_waitcnt vmcnt(40)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v32
; GFX900-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:108
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:104
; GFX900-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
; GFX900-NEXT: v_cvt_f64_f32_e32 v[17:18], v2
-; GFX900-NEXT: s_waitcnt vmcnt(41)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v33
; GFX900-NEXT: v_cvt_f64_f32_e32 v[19:20], v2
-; GFX900-NEXT: s_waitcnt vmcnt(40)
+; GFX900-NEXT: s_waitcnt vmcnt(44)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v34
+; GFX900-NEXT: s_waitcnt vmcnt(43)
; GFX900-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX900-NEXT: v_cvt_f64_f32_e32 v[21:22], v2
+; GFX900-NEXT: s_waitcnt vmcnt(41)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX900-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:100
; GFX900-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:96
; GFX900-NEXT: v_cvt_f64_f32_e32 v[8:9], v6
; GFX900-NEXT: v_cvt_f64_f32_e32 v[5:6], v2
-; GFX900-NEXT: s_waitcnt vmcnt(41)
; GFX900-NEXT: v_lshlrev_b32_e32 v10, 16, v7
; GFX900-NEXT: s_waitcnt vmcnt(40)
; GFX900-NEXT: v_lshlrev_b32_e32 v7, 16, v1
@@ -8527,85 +8527,87 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: v_mov_b32_e32 v2, v1
; GFX950-NEXT: global_load_ushort v1, v[2:3], off offset:2
-; GFX950-NEXT: global_load_ushort v4, v[2:3], off offset:12
-; GFX950-NEXT: global_load_ushort v5, v[2:3], off offset:8
-; GFX950-NEXT: global_load_ushort v6, v[2:3], off offset:4
; GFX950-NEXT: global_load_ushort v7, v[2:3], off
; GFX950-NEXT: global_load_ushort v8, v[2:3], off offset:6
+; GFX950-NEXT: global_load_ushort v6, v[2:3], off offset:4
; GFX950-NEXT: global_load_ushort v9, v[2:3], off offset:10
+; GFX950-NEXT: global_load_ushort v5, v[2:3], off offset:8
; GFX950-NEXT: global_load_ushort v10, v[2:3], off offset:14
+; GFX950-NEXT: global_load_ushort v4, v[2:3], off offset:12
; GFX950-NEXT: global_load_ushort v11, v[2:3], off offset:18
-; GFX950-NEXT: global_load_ushort v12, v[2:3], off offset:28
-; GFX950-NEXT: global_load_ushort v13, v[2:3], off offset:24
-; GFX950-NEXT: global_load_ushort v14, v[2:3], off offset:20
; GFX950-NEXT: global_load_ushort v15, v[2:3], off offset:16
; GFX950-NEXT: global_load_ushort v16, v[2:3], off offset:22
+; GFX950-NEXT: global_load_ushort v14, v[2:3], off offset:20
; GFX950-NEXT: global_load_ushort v17, v[2:3], off offset:26
+; GFX950-NEXT: global_load_ushort v13, v[2:3], off offset:24
; GFX950-NEXT: global_load_ushort v18, v[2:3], off offset:30
+; GFX950-NEXT: global_load_ushort v12, v[2:3], off offset:28
; GFX950-NEXT: global_load_ushort v19, v[2:3], off offset:34
-; GFX950-NEXT: global_load_ushort v20, v[2:3], off offset:44
-; GFX950-NEXT: global_load_ushort v21, v[2:3], off offset:40
-; GFX950-NEXT: global_load_ushort v22, v[2:3], off offset:36
; GFX950-NEXT: global_load_ushort v23, v[2:3], off offset:32
; GFX950-NEXT: global_load_ushort v24, v[2:3], off offset:38
+; GFX950-NEXT: global_load_ushort v22, v[2:3], off offset:36
; GFX950-NEXT: global_load_ushort v25, v[2:3], off offset:42
+; GFX950-NEXT: global_load_ushort v21, v[2:3], off offset:40
; GFX950-NEXT: global_load_ushort v26, v[2:3], off offset:46
+; GFX950-NEXT: global_load_ushort v20, v[2:3], off offset:44
; GFX950-NEXT: global_load_ushort v42, v[2:3], off offset:50
; GFX950-NEXT: global_load_ushort v43, v[2:3], off offset:62
; GFX950-NEXT: global_load_ushort v46, v[2:3], off offset:60
-; GFX950-NEXT: global_load_ushort v47, v[2:3], off offset:56
-; GFX950-NEXT: global_load_ushort v60, v[2:3], off offset:52
; GFX950-NEXT: global_load_ushort v56, v[2:3], off offset:48
; GFX950-NEXT: global_load_ushort v57, v[2:3], off offset:54
; GFX950-NEXT: global_load_ushort v58, v[2:3], off offset:58
+; GFX950-NEXT: global_load_ushort v47, v[2:3], off offset:56
+; GFX950-NEXT: global_load_ushort v60, v[2:3], off offset:52
; GFX950-NEXT: s_waitcnt vmcnt(31)
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX950-NEXT: s_waitcnt vmcnt(30)
-; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v4
+; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX950-NEXT: s_waitcnt vmcnt(29)
-; GFX950-NEXT: v_lshlrev_b32_e32 v28, 16, v5
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[4:5], v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX950-NEXT: s_waitcnt vmcnt(28)
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX950-NEXT: s_waitcnt vmcnt(27)
-; GFX950-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; GFX950-NEXT: v_lshlrev_b32_e32 v27, 16, v9
; GFX950-NEXT: s_waitcnt vmcnt(26)
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX950-NEXT: v_lshlrev_b32_e32 v28, 16, v5
; GFX950-NEXT: s_waitcnt vmcnt(25)
-; GFX950-NEXT: v_lshlrev_b32_e32 v27, 16, v9
-; GFX950-NEXT: s_waitcnt vmcnt(24)
; GFX950-NEXT: v_lshlrev_b32_e32 v29, 16, v10
+; GFX950-NEXT: s_waitcnt vmcnt(24)
+; GFX950-NEXT: v_lshlrev_b32_e32 v30, 16, v4
; GFX950-NEXT: s_waitcnt vmcnt(23)
; GFX950-NEXT: v_lshlrev_b32_e32 v31, 16, v11
; GFX950-NEXT: s_waitcnt vmcnt(22)
-; GFX950-NEXT: v_lshlrev_b32_e32 v38, 16, v12
+; GFX950-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX950-NEXT: s_waitcnt vmcnt(21)
-; GFX950-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX950-NEXT: v_lshlrev_b32_e32 v33, 16, v16
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[4:5], v1
; GFX950-NEXT: s_waitcnt vmcnt(20)
; GFX950-NEXT: v_lshlrev_b32_e32 v34, 16, v14
-; GFX950-NEXT: s_waitcnt vmcnt(19)
-; GFX950-NEXT: v_lshlrev_b32_e32 v32, 16, v15
; GFX950-NEXT: s_waitcnt vmcnt(18)
-; GFX950-NEXT: v_lshlrev_b32_e32 v33, 16, v16
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[12:13], v27
-; GFX950-NEXT: s_waitcnt vmcnt(16)
+; GFX950-NEXT: v_lshlrev_b32_e32 v36, 16, v13
+; GFX950-NEXT: s_waitcnt vmcnt(17)
; GFX950-NEXT: v_lshlrev_b32_e32 v37, 16, v18
+; GFX950-NEXT: s_waitcnt vmcnt(16)
+; GFX950-NEXT: v_lshlrev_b32_e32 v38, 16, v12
; GFX950-NEXT: s_waitcnt vmcnt(15)
; GFX950-NEXT: v_lshlrev_b32_e32 v39, 16, v19
-; GFX950-NEXT: s_waitcnt vmcnt(14)
-; GFX950-NEXT: v_lshlrev_b32_e32 v44, 16, v20
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[12:13], v27
; GFX950-NEXT: s_waitcnt vmcnt(13)
-; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v21
+; GFX950-NEXT: v_lshlrev_b32_e32 v49, 16, v24
; GFX950-NEXT: v_cvt_f64_f32_e32 v[14:15], v30
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[20:21], v31
+; GFX950-NEXT: s_waitcnt vmcnt(11)
+; GFX950-NEXT: v_lshlrev_b32_e32 v53, 16, v25
; GFX950-NEXT: s_waitcnt vmcnt(10)
-; GFX950-NEXT: v_lshlrev_b32_e32 v49, 16, v24
+; GFX950-NEXT: v_lshlrev_b32_e32 v40, 16, v21
; GFX950-NEXT: s_waitcnt vmcnt(9)
-; GFX950-NEXT: v_lshlrev_b32_e32 v53, 16, v25
-; GFX950-NEXT: s_waitcnt vmcnt(8)
; GFX950-NEXT: v_lshlrev_b32_e32 v41, 16, v26
+; GFX950-NEXT: s_waitcnt vmcnt(8)
+; GFX950-NEXT: v_lshlrev_b32_e32 v44, 16, v20
; GFX950-NEXT: s_waitcnt vmcnt(7)
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v42
; GFX950-NEXT: s_waitcnt vmcnt(6)
; GFX950-NEXT: v_lshlrev_b32_e32 v42, 16, v43
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[20:21], v31
; GFX950-NEXT: v_cvt_f64_f32_e32 v[18:19], v32
; GFX950-NEXT: v_cvt_f64_f32_e32 v[24:25], v33
; GFX950-NEXT: v_cvt_f64_f32_e32 v[26:27], v36
@@ -8617,10 +8619,11 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX950-NEXT: s_waitcnt vmcnt(5)
; GFX950-NEXT: v_lshlrev_b32_e32 v42, 16, v46
; GFX950-NEXT: v_cvt_f64_f32_e32 v[42:43], v42
-; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: s_waitcnt vmcnt(2)
; GFX950-NEXT: v_lshlrev_b32_e32 v46, 16, v58
; GFX950-NEXT: scratch_store_dwordx4 v0, v[42:45], off offset:240
; GFX950-NEXT: v_cvt_f64_f32_e32 v[58:59], v46
+; GFX950-NEXT: s_waitcnt vmcnt(2)
; GFX950-NEXT: v_lshlrev_b32_e32 v46, 16, v47
; GFX950-NEXT: v_cvt_f64_f32_e32 v[44:45], v1
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v56
@@ -8631,11 +8634,11 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX950-NEXT: v_lshlrev_b32_e32 v48, 16, v23
; GFX950-NEXT: v_lshlrev_b32_e32 v52, 16, v22
; GFX950-NEXT: scratch_store_dwordx4 v0, v[56:59], off offset:224
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX950-NEXT: v_cvt_f64_f32_e32 v[10:11], v28
+; GFX950-NEXT: v_cvt_f64_f32_e32 v[16:17], v29
; GFX950-NEXT: v_cvt_f64_f32_e32 v[58:59], v1
+; GFX950-NEXT: s_waitcnt vmcnt(2)
; GFX950-NEXT: v_lshlrev_b32_e32 v1, 16, v60
-; GFX950-NEXT: v_cvt_f64_f32_e32 v[16:17], v29
; GFX950-NEXT: v_cvt_f64_f32_e32 v[22:23], v34
; GFX950-NEXT: v_cvt_f64_f32_e32 v[28:29], v35
; GFX950-NEXT: v_cvt_f64_f32_e32 v[34:35], v48
@@ -8876,105 +8879,105 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1f
; GFX11-NEXT: global_load_u16 v3, v[1:2], off offset:2
-; GFX11-NEXT: global_load_u16 v4, v[1:2], off offset:12
-; GFX11-NEXT: global_load_u16 v5, v[1:2], off offset:8
-; GFX11-NEXT: global_load_u16 v6, v[1:2], off offset:4
; GFX11-NEXT: global_load_u16 v7, v[1:2], off
; GFX11-NEXT: global_load_u16 v8, v[1:2], off offset:6
+; GFX11-NEXT: global_load_u16 v6, v[1:2], off offset:4
; GFX11-NEXT: global_load_u16 v9, v[1:2], off offset:10
+; GFX11-NEXT: global_load_u16 v5, v[1:2], off offset:8
; GFX11-NEXT: global_load_u16 v10, v[1:2], off offset:14
+; GFX11-NEXT: global_load_u16 v4, v[1:2], off offset:12
; GFX11-NEXT: global_load_u16 v11, v[1:2], off offset:18
-; GFX11-NEXT: global_load_u16 v12, v[1:2], off offset:28
-; GFX11-NEXT: global_load_u16 v13, v[1:2], off offset:24
-; GFX11-NEXT: global_load_u16 v14, v[1:2], off offset:20
; GFX11-NEXT: global_load_u16 v15, v[1:2], off offset:16
; GFX11-NEXT: global_load_u16 v16, v[1:2], off offset:22
+; GFX11-NEXT: global_load_u16 v14, v[1:2], off offset:20
; GFX11-NEXT: global_load_u16 v17, v[1:2], off offset:26
+; GFX11-NEXT: global_load_u16 v13, v[1:2], off offset:24
; GFX11-NEXT: global_load_u16 v18, v[1:2], off offset:30
+; GFX11-NEXT: global_load_u16 v12, v[1:2], off offset:28
; GFX11-NEXT: global_load_u16 v19, v[1:2], off offset:34
-; GFX11-NEXT: global_load_u16 v20, v[1:2], off offset:44
-; GFX11-NEXT: global_load_u16 v21, v[1:2], off offset:40
-; GFX11-NEXT: global_load_u16 v22, v[1:2], off offset:36
; GFX11-NEXT: global_load_u16 v23, v[1:2], off offset:32
; GFX11-NEXT: global_load_u16 v24, v[1:2], off offset:38
+; GFX11-NEXT: global_load_u16 v22, v[1:2], off offset:36
; GFX11-NEXT: global_load_u16 v25, v[1:2], off offset:42
+; GFX11-NEXT: global_load_u16 v21, v[1:2], off offset:40
; GFX11-NEXT: global_load_u16 v26, v[1:2], off offset:46
+; GFX11-NEXT: global_load_u16 v20, v[1:2], off offset:44
; GFX11-NEXT: global_load_u16 v27, v[1:2], off offset:50
-; GFX11-NEXT: global_load_u16 v28, v[1:2], off offset:60
-; GFX11-NEXT: global_load_u16 v29, v[1:2], off offset:56
-; GFX11-NEXT: global_load_u16 v30, v[1:2], off offset:52
; GFX11-NEXT: global_load_u16 v31, v[1:2], off offset:48
; GFX11-NEXT: global_load_u16 v32, v[1:2], off offset:54
+; GFX11-NEXT: global_load_u16 v30, v[1:2], off offset:52
; GFX11-NEXT: global_load_u16 v33, v[1:2], off offset:58
+; GFX11-NEXT: global_load_u16 v29, v[1:2], off offset:56
+; GFX11-NEXT: global_load_u16 v28, v[1:2], off offset:60
; GFX11-NEXT: global_load_u16 v1, v[1:2], off offset:62
; GFX11-NEXT: s_waitcnt vmcnt(31)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-NEXT: s_waitcnt vmcnt(30)
-; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v7
; GFX11-NEXT: s_waitcnt vmcnt(29)
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; GFX11-NEXT: s_waitcnt vmcnt(28)
; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-NEXT: s_waitcnt vmcnt(27)
-; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11-NEXT: s_waitcnt vmcnt(26)
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11-NEXT: s_waitcnt vmcnt(25)
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-NEXT: s_waitcnt vmcnt(24)
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-NEXT: s_waitcnt vmcnt(24)
+; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v4
; GFX11-NEXT: s_waitcnt vmcnt(23)
; GFX11-NEXT: v_lshlrev_b32_e32 v34, 16, v11
; GFX11-NEXT: s_waitcnt vmcnt(22)
-; GFX11-NEXT: v_lshlrev_b32_e32 v100, 16, v12
+; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v15
; GFX11-NEXT: s_waitcnt vmcnt(21)
-; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v16
; GFX11-NEXT: s_waitcnt vmcnt(20)
; GFX11-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX11-NEXT: s_waitcnt vmcnt(19)
-; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v15
+; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX11-NEXT: s_waitcnt vmcnt(18)
-; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v16
+; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX11-NEXT: s_waitcnt vmcnt(17)
-; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-NEXT: s_waitcnt vmcnt(16)
; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-NEXT: s_waitcnt vmcnt(16)
+; GFX11-NEXT: v_lshlrev_b32_e32 v100, 16, v12
; GFX11-NEXT: s_waitcnt vmcnt(15)
; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v19
; GFX11-NEXT: s_waitcnt vmcnt(14)
-; GFX11-NEXT: v_lshlrev_b32_e32 v49, 16, v20
+; GFX11-NEXT: v_lshlrev_b32_e32 v101, 16, v23
; GFX11-NEXT: s_waitcnt vmcnt(13)
-; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v24
; GFX11-NEXT: s_waitcnt vmcnt(12)
; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX11-NEXT: s_waitcnt vmcnt(11)
-; GFX11-NEXT: v_lshlrev_b32_e32 v101, 16, v23
+; GFX11-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; GFX11-NEXT: s_waitcnt vmcnt(10)
-; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v24
+; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; GFX11-NEXT: s_waitcnt vmcnt(9)
-; GFX11-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-NEXT: s_waitcnt vmcnt(8)
; GFX11-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-NEXT: s_waitcnt vmcnt(8)
+; GFX11-NEXT: v_lshlrev_b32_e32 v49, 16, v20
; GFX11-NEXT: s_waitcnt vmcnt(7)
; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v27
; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v28
+; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v31
; GFX11-NEXT: s_waitcnt vmcnt(5)
-; GFX11-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v32
; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v31
+; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v32
+; GFX11-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v28
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_cvt_f64_f32_e32 v[96:97], v65
-; GFX11-NEXT: v_cvt_f64_f32_e32 v[84:85], v29
; GFX11-NEXT: v_cvt_f64_f32_e32 v[82:83], v64
; GFX11-NEXT: v_cvt_f64_f32_e32 v[86:87], v33
+; GFX11-NEXT: v_cvt_f64_f32_e32 v[84:85], v29
+; GFX11-NEXT: v_cvt_f64_f32_e32 v[96:97], v65
; GFX11-NEXT: v_cvt_f64_f32_e32 v[98:99], v1
; GFX11-NEXT: v_cvt_f64_f32_e32 v[80:81], v30
; GFX11-NEXT: v_cvt_f64_f32_e32 v[70:71], v52
@@ -9029,70 +9032,74 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-NEXT: s_clause 0x1f
; GFX1250-NEXT: global_load_u16 v1, v[2:3], off offset:2
-; GFX1250-NEXT: global_load_u16 v10, v[2:3], off offset:12
-; GFX1250-NEXT: global_load_u16 v6, v[2:3], off offset:8
-; GFX1250-NEXT: global_load_u16 v4, v[2:3], off offset:4
; GFX1250-NEXT: global_load_u16 v5, v[2:3], off
; GFX1250-NEXT: global_load_u16 v7, v[2:3], off offset:6
; GFX1250-NEXT: global_load_u16 v8, v[2:3], off offset:62
; GFX1250-NEXT: global_load_u16 v9, v[2:3], off offset:60
+; GFX1250-NEXT: global_load_u16 v4, v[2:3], off offset:4
; GFX1250-NEXT: global_load_u16 v11, v[2:3], off offset:58
; GFX1250-NEXT: global_load_u16 v12, v[2:3], off offset:56
; GFX1250-NEXT: global_load_u16 v13, v[2:3], off offset:10
+; GFX1250-NEXT: global_load_u16 v6, v[2:3], off offset:8
; GFX1250-NEXT: global_load_u16 v14, v[2:3], off offset:14
+; GFX1250-NEXT: global_load_u16 v10, v[2:3], off offset:12
; GFX1250-NEXT: global_load_u16 v15, v[2:3], off offset:18
-; GFX1250-NEXT: global_load_u16 v16, v[2:3], off offset:28
-; GFX1250-NEXT: global_load_u16 v17, v[2:3], off offset:24
-; GFX1250-NEXT: global_load_u16 v18, v[2:3], off offset:20
; GFX1250-NEXT: global_load_u16 v19, v[2:3], off offset:16
; GFX1250-NEXT: global_load_u16 v20, v[2:3], off offset:22
+; GFX1250-NEXT: global_load_u16 v18, v[2:3], off offset:20
; GFX1250-NEXT: global_load_u16 v21, v[2:3], off offset:26
+; GFX1250-NEXT: global_load_u16 v17, v[2:3], off offset:24
; GFX1250-NEXT: global_load_u16 v22, v[2:3], off offset:30
+; GFX1250-NEXT: global_load_u16 v16, v[2:3], off offset:28
; GFX1250-NEXT: global_load_u16 v23, v[2:3], off offset:34
-; GFX1250-NEXT: global_load_u16 v24, v[2:3], off offset:44
-; GFX1250-NEXT: global_load_u16 v25, v[2:3], off offset:40
-; GFX1250-NEXT: global_load_u16 v26, v[2:3], off offset:36
; GFX1250-NEXT: global_load_u16 v27, v[2:3], off offset:32
; GFX1250-NEXT: global_load_u16 v28, v[2:3], off offset:38
+; GFX1250-NEXT: global_load_u16 v26, v[2:3], off offset:36
; GFX1250-NEXT: global_load_u16 v29, v[2:3], off offset:42
+; GFX1250-NEXT: global_load_u16 v25, v[2:3], off offset:40
; GFX1250-NEXT: global_load_u16 v30, v[2:3], off offset:46
+; GFX1250-NEXT: global_load_u16 v24, v[2:3], off offset:44
; GFX1250-NEXT: global_load_u16 v31, v[2:3], off offset:50
-; GFX1250-NEXT: global_load_u16 v32, v[2:3], off offset:52
; GFX1250-NEXT: global_load_u16 v33, v[2:3], off offset:48
; GFX1250-NEXT: global_load_u16 v34, v[2:3], off offset:54
-; GFX1250-NEXT: s_wait_loadcnt 0x1e
-; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v37, 16, v10
+; GFX1250-NEXT: global_load_u16 v32, v[2:3], off offset:52
+; GFX1250-NEXT: s_wait_loadcnt 0x1f
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1250-NEXT: s_wait_loadcnt 0x1d
+; GFX1250-NEXT: v_dual_lshlrev_b32 v84, 16, v5 :: v_dual_lshlrev_b32 v35, 16, v7
; GFX1250-NEXT: s_wait_loadcnt 0x1b
-; GFX1250-NEXT: v_dual_lshlrev_b32 v85, 16, v4 :: v_dual_lshlrev_b32 v84, 16, v5
-; GFX1250-NEXT: s_wait_loadcnt 0x19
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v35, 16, v7 :: v_dual_lshlrev_b32 v2, 16, v8
-; GFX1250-NEXT: s_wait_loadcnt 0x17
-; GFX1250-NEXT: v_dual_lshlrev_b32 v3, 16, v9 :: v_dual_lshlrev_b32 v7, 16, v11
-; GFX1250-NEXT: s_wait_loadcnt 0x15
-; GFX1250-NEXT: v_dual_lshlrev_b32 v11, 16, v12 :: v_dual_lshlrev_b32 v12, 16, v13
-; GFX1250-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v8 :: v_dual_lshlrev_b32 v3, 16, v9
+; GFX1250-NEXT: s_wait_loadcnt 0x19
+; GFX1250-NEXT: v_dual_lshlrev_b32 v85, 16, v4 :: v_dual_lshlrev_b32 v7, 16, v11
+; GFX1250-NEXT: s_wait_loadcnt 0x18
+; GFX1250-NEXT: v_lshlrev_b32_e32 v11, 16, v12
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v2
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
+; GFX1250-NEXT: s_wait_loadcnt 0x16
+; GFX1250-NEXT: v_dual_lshlrev_b32 v12, 16, v13 :: v_dual_lshlrev_b32 v13, 16, v6
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v7
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v11
; GFX1250-NEXT: s_wait_loadcnt 0x13
; GFX1250-NEXT: v_dual_lshlrev_b32 v36, 16, v14 :: v_dual_lshlrev_b32 v38, 16, v15
-; GFX1250-NEXT: s_wait_loadcnt 0xc
-; GFX1250-NEXT: v_dual_lshlrev_b32 v21, 16, v21 :: v_dual_lshlrev_b32 v39, 16, v22
+; GFX1250-NEXT: s_wait_loadcnt 0xf
+; GFX1250-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX1250-NEXT: s_wait_loadcnt 0xd
+; GFX1250-NEXT: v_dual_lshlrev_b32 v68, 16, v17 :: v_dual_lshlrev_b32 v39, 16, v22
; GFX1250-NEXT: s_wait_loadcnt 0xb
-; GFX1250-NEXT: v_dual_lshlrev_b32 v49, 16, v23 :: v_dual_lshlrev_b32 v68, 16, v17
-; GFX1250-NEXT: s_wait_loadcnt 0x9
-; GFX1250-NEXT: v_dual_lshlrev_b32 v48, 16, v16 :: v_dual_lshlrev_b32 v25, 16, v25
-; GFX1250-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX1250-NEXT: s_wait_loadcnt 0x5
+; GFX1250-NEXT: v_dual_lshlrev_b32 v48, 16, v16 :: v_dual_lshlrev_b32 v49, 16, v23
+; GFX1250-NEXT: v_dual_lshlrev_b32 v37, 16, v10 :: v_dual_lshlrev_b32 v80, 16, v19
+; GFX1250-NEXT: s_wait_loadcnt 0x7
; GFX1250-NEXT: v_dual_lshlrev_b32 v50, 16, v28 :: v_dual_lshlrev_b32 v64, 16, v29
+; GFX1250-NEXT: s_wait_loadcnt 0x5
+; GFX1250-NEXT: v_dual_lshlrev_b32 v25, 16, v25 :: v_dual_lshlrev_b32 v51, 16, v30
; GFX1250-NEXT: s_wait_loadcnt 0x3
-; GFX1250-NEXT: v_dual_lshlrev_b32 v51, 16, v30 :: v_dual_lshlrev_b32 v52, 16, v31
-; GFX1250-NEXT: v_dual_lshlrev_b32 v69, 16, v27 :: v_dual_lshlrev_b32 v70, 16, v26
+; GFX1250-NEXT: v_dual_lshlrev_b32 v24, 16, v24 :: v_dual_lshlrev_b32 v52, 16, v31
+; GFX1250-NEXT: s_wait_loadcnt 0x1
+; GFX1250-NEXT: v_dual_lshlrev_b32 v33, 16, v33 :: v_dual_lshlrev_b32 v53, 16, v34
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshlrev_b32 v53, 16, v34 :: v_dual_lshlrev_b32 v32, 16, v32
-; GFX1250-NEXT: v_lshlrev_b32_e32 v33, 16, v33
+; GFX1250-NEXT: v_dual_lshlrev_b32 v32, 16, v32 :: v_dual_lshlrev_b32 v69, 16, v27
+; GFX1250-NEXT: v_lshlrev_b32_e32 v70, 16, v26
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[22:23], v38
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[30:31], v39
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[38:39], v50
@@ -9104,16 +9111,15 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0
; GFX1250-NEXT: v_dual_lshlrev_b32 v20, 16, v20 :: v_dual_lshlrev_b32 v81, 16, v18
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[66:67], v64
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[64:65], v25
-; GFX1250-NEXT: v_lshlrev_b32_e32 v80, 16, v19
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[18:19], v36
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[16:17], v37
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[36:37], v70
; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off offset:240
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v51
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[50:51], v52
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[52:53], v32
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v24
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[18:19], v36
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[16:17], v37
+; GFX1250-NEXT: v_cvt_f64_f32_e32 v[36:37], v70
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[32:33], v69
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[70:71], v21
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[68:69], v68
@@ -43978,39 +43984,39 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_and_b32_e32 v0, 1, v0
; GCN-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0
-; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
-; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
-; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
-; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72
-; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
-; GCN-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
-; GCN-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:16
-; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80
-; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
-; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:84
-; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:24
-; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88
-; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
-; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:92
-; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:32
-; GCN-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96
-; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36
-; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:40
-; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104
-; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:44
-; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108
-; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:48
-; GCN-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
-; GCN-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:52
-; GCN-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116
-; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:56
-; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120
-; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60
-; GCN-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:124
; GCN-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64
; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60
+; GCN-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:124
+; GCN-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:56
+; GCN-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120
+; GCN-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:52
+; GCN-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116
+; GCN-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:48
+; GCN-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
+; GCN-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:44
+; GCN-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108
+; GCN-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:40
+; GCN-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104
+; GCN-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36
+; GCN-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
+; GCN-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:32
+; GCN-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96
+; GCN-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
+; GCN-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:92
+; GCN-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GCN-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88
+; GCN-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
+; GCN-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:84
+; GCN-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:16
+; GCN-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80
+; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
+; GCN-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
+; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72
+; GCN-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
+; GCN-NEXT: s_waitcnt vmcnt(14)
; GCN-NEXT: v_cndmask_b32_e64 v30, v32, v31, s[34:35]
; GCN-NEXT: v_cndmask_b32_e64 v31, v32, v31, s[30:31]
; GCN-NEXT: v_cndmask_b32_e64 v32, v29, v28, s[94:95]
@@ -44029,18 +44035,25 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GCN-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[44:45]
; GCN-NEXT: v_cndmask_b32_e64 v17, v15, v14, s[42:43]
; GCN-NEXT: v_cndmask_b32_e64 v14, v15, v14, s[40:41]
+; GCN-NEXT: s_waitcnt vmcnt(12)
; GCN-NEXT: v_cndmask_b32_e64 v15, v13, v12, s[28:29]
; GCN-NEXT: v_cndmask_b32_e64 v12, v13, v12, s[26:27]
+; GCN-NEXT: s_waitcnt vmcnt(10)
; GCN-NEXT: v_cndmask_b32_e64 v13, v11, v10, s[24:25]
; GCN-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[22:23]
+; GCN-NEXT: s_waitcnt vmcnt(8)
; GCN-NEXT: v_cndmask_b32_e64 v11, v9, v8, s[20:21]
; GCN-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[18:19]
+; GCN-NEXT: s_waitcnt vmcnt(6)
; GCN-NEXT: v_cndmask_b32_e64 v9, v7, v6, s[16:17]
; GCN-NEXT: v_cndmask_b32_e64 v6, v7, v6, s[14:15]
+; GCN-NEXT: s_waitcnt vmcnt(4)
; GCN-NEXT: v_cndmask_b32_e64 v7, v5, v4, s[12:13]
; GCN-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[10:11]
+; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_cndmask_b32_e64 v5, v3, v2, s[8:9]
; GCN-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[6:7]
+; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_cndmask_b32_e64 v3, v1, v0, s[4:5]
; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GCN-NEXT: s_mov_b32 s4, 0xffff
@@ -44146,39 +44159,39 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
; GFX7-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0
-; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
-; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
-; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
-; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72
-; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
-; GFX7-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
-; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:16
-; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80
-; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
-; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:84
-; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:24
-; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88
-; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
-; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:92
-; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:32
-; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96
-; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36
-; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
-; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:40
-; GFX7-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104
-; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:44
-; GFX7-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108
-; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:48
-; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
-; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:52
-; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116
-; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:56
-; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120
-; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60
-; GFX7-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:124
; GFX7-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64
; GFX7-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128
-; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60
+; GFX7-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:124
+; GFX7-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:56
+; GFX7-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:120
+; GFX7-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:52
+; GFX7-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:116
+; GFX7-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:48
+; GFX7-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
+; GFX7-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:44
+; GFX7-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:108
+; GFX7-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:40
+; GFX7-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:104
+; GFX7-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36
+; GFX7-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100
+; GFX7-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:32
+; GFX7-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96
+; GFX7-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:28
+; GFX7-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:92
+; GFX7-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:24
+; GFX7-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88
+; GFX7-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20
+; GFX7-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:84
+; GFX7-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:16
+; GFX7-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80
+; GFX7-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:12
+; GFX7-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76
+; GFX7-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8
+; GFX7-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72
+; GFX7-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4
+; GFX7-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68
+; GFX7-NEXT: s_waitcnt vmcnt(14)
; GFX7-NEXT: v_cndmask_b32_e64 v30, v32, v31, s[34:35]
; GFX7-NEXT: v_cndmask_b32_e64 v31, v32, v31, s[30:31]
; GFX7-NEXT: v_cndmask_b32_e64 v32, v29, v28, s[94:95]
@@ -44197,18 +44210,25 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX7-NEXT: v_cndmask_b32_e64 v16, v17, v16, s[44:45]
; GFX7-NEXT: v_cndmask_b32_e64 v17, v15, v14, s[42:43]
; GFX7-NEXT: v_cndmask_b32_e64 v14, v15, v14, s[40:41]
+; GFX7-NEXT: s_waitcnt vmcnt(12)
; GFX7-NEXT: v_cndmask_b32_e64 v15, v13, v12, s[28:29]
; GFX7-NEXT: v_cndmask_b32_e64 v12, v13, v12, s[26:27]
+; GFX7-NEXT: s_waitcnt vmcnt(10)
; GFX7-NEXT: v_cndmask_b32_e64 v13, v11, v10, s[24:25]
; GFX7-NEXT: v_cndmask_b32_e64 v10, v11, v10, s[22:23]
+; GFX7-NEXT: s_waitcnt vmcnt(8)
; GFX7-NEXT: v_cndmask_b32_e64 v11, v9, v8, s[20:21]
; GFX7-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[18:19]
+; GFX7-NEXT: s_waitcnt vmcnt(6)
; GFX7-NEXT: v_cndmask_b32_e64 v9, v7, v6, s[16:17]
; GFX7-NEXT: v_cndmask_b32_e64 v6, v7, v6, s[14:15]
+; GFX7-NEXT: s_waitcnt vmcnt(4)
; GFX7-NEXT: v_cndmask_b32_e64 v7, v5, v4, s[12:13]
; GFX7-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[10:11]
+; GFX7-NEXT: s_waitcnt vmcnt(2)
; GFX7-NEXT: v_cndmask_b32_e64 v5, v3, v2, s[8:9]
; GFX7-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[6:7]
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cndmask_b32_e64 v3, v1, v0, s[4:5]
; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX7-NEXT: s_mov_b32 s4, 0xffff
@@ -50852,7 +50872,6 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x10
-; GFX1250-NEXT: scratch_load_b32 v31, off, s32 offset:64
; GFX1250-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX1250-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX1250-NEXT: scratch_load_b32 v34, off, s32 offset:12
@@ -50868,36 +50887,37 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX1250-NEXT: scratch_load_b32 v52, off, s32 offset:52
; GFX1250-NEXT: scratch_load_b32 v53, off, s32 offset:56
; GFX1250-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX1250-NEXT: scratch_load_b32 v31, off, s32 offset:64
; GFX1250-NEXT: scratch_load_b32 v55, off, s32
-; GFX1250-NEXT: s_wait_loadcnt 0xf
+; GFX1250-NEXT: s_wait_loadcnt 0x10
; GFX1250-NEXT: v_pk_fma_bf16 v0, v0, v16, v32
-; GFX1250-NEXT: s_wait_loadcnt 0xe
+; GFX1250-NEXT: s_wait_loadcnt 0xf
; GFX1250-NEXT: v_pk_fma_bf16 v1, v1, v17, v33
-; GFX1250-NEXT: s_wait_loadcnt 0xd
+; GFX1250-NEXT: s_wait_loadcnt 0xe
; GFX1250-NEXT: v_pk_fma_bf16 v2, v2, v18, v34
-; GFX1250-NEXT: s_wait_loadcnt 0xc
+; GFX1250-NEXT: s_wait_loadcnt 0xd
; GFX1250-NEXT: v_pk_fma_bf16 v3, v3, v19, v35
-; GFX1250-NEXT: s_wait_loadcnt 0xb
+; GFX1250-NEXT: s_wait_loadcnt 0xc
; GFX1250-NEXT: v_pk_fma_bf16 v4, v4, v20, v36
-; GFX1250-NEXT: s_wait_loadcnt 0xa
+; GFX1250-NEXT: s_wait_loadcnt 0xb
; GFX1250-NEXT: v_pk_fma_bf16 v5, v5, v21, v37
-; GFX1250-NEXT: s_wait_loadcnt 0x9
+; GFX1250-NEXT: s_wait_loadcnt 0xa
; GFX1250-NEXT: v_pk_fma_bf16 v6, v6, v22, v38
-; GFX1250-NEXT: s_wait_loadcnt 0x8
+; GFX1250-NEXT: s_wait_loadcnt 0x9
; GFX1250-NEXT: v_pk_fma_bf16 v7, v7, v23, v39
-; GFX1250-NEXT: s_wait_loadcnt 0x7
+; GFX1250-NEXT: s_wait_loadcnt 0x8
; GFX1250-NEXT: v_pk_fma_bf16 v8, v8, v24, v48
-; GFX1250-NEXT: s_wait_loadcnt 0x6
+; GFX1250-NEXT: s_wait_loadcnt 0x7
; GFX1250-NEXT: v_pk_fma_bf16 v9, v9, v25, v49
-; GFX1250-NEXT: s_wait_loadcnt 0x5
+; GFX1250-NEXT: s_wait_loadcnt 0x6
; GFX1250-NEXT: v_pk_fma_bf16 v10, v10, v26, v50
-; GFX1250-NEXT: s_wait_loadcnt 0x4
+; GFX1250-NEXT: s_wait_loadcnt 0x5
; GFX1250-NEXT: v_pk_fma_bf16 v11, v11, v27, v51
-; GFX1250-NEXT: s_wait_loadcnt 0x3
+; GFX1250-NEXT: s_wait_loadcnt 0x4
; GFX1250-NEXT: v_pk_fma_bf16 v12, v12, v28, v52
-; GFX1250-NEXT: s_wait_loadcnt 0x2
+; GFX1250-NEXT: s_wait_loadcnt 0x3
; GFX1250-NEXT: v_pk_fma_bf16 v13, v13, v29, v53
-; GFX1250-NEXT: s_wait_loadcnt 0x1
+; GFX1250-NEXT: s_wait_loadcnt 0x2
; GFX1250-NEXT: v_pk_fma_bf16 v14, v14, v30, v54
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_pk_fma_bf16 v15, v15, v55, v31
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index 70106cf0838cd..f41eb56318b98 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -401,14 +401,14 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x2000
-; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(2)
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a0, v15 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
@@ -427,8 +427,9 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7)
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
-; GISEL-GFX942-NEXT: s_nop 0
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(14)
; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(14)
; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
@@ -922,14 +923,14 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-GFX942-NEXT: s_add_i32 s1, s0, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v60, s1
+; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[8:11], v60, s[8:11], 0 offen
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[4:7], v60, s[8:11], 0 offen offset:16
-; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[12:15], v60, s[8:11], 0 offen offset:32
; GISEL-GFX942-NEXT: s_add_i32 s2, s12, s16
; GISEL-GFX942-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX942-NEXT: s_addk_i32 s16, 0x100
; GISEL-GFX942-NEXT: s_cmpk_lt_u32 s16, 0x100
-; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(2)
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a0, v15 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v14 ; Reload Reuse
; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v13 ; Reload Reuse
@@ -948,8 +949,9 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[56:59], v60, s[8:11], 0 offen offset:224
; GISEL-GFX942-NEXT: s_nop 0
; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[60:63], v60, s[8:11], 0 offen offset:240
-; GISEL-GFX942-NEXT: s_nop 0
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(14)
; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[8:11], v0, s[4:7], 0 offen
+; GISEL-GFX942-NEXT: s_waitcnt vmcnt(14)
; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[4:7], v0, s[4:7], 0 offen offset:16
; GISEL-GFX942-NEXT: s_nop 1
; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index eb3816ec1cbc0..84dc52ce7bef5 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -318,13 +318,13 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
; COEXEC-NEXT: s_wait_dscnt 0x13
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
-; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
+; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x16
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
@@ -337,11 +337,11 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
-; COEXEC-NEXT: s_wait_dscnt 0x6
+; COEXEC-NEXT: s_wait_dscnt 0x7
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
-; COEXEC-NEXT: s_wait_dscnt 0x4
+; COEXEC-NEXT: s_wait_dscnt 0x5
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
-; COEXEC-NEXT: s_wait_dscnt 0x2
+; COEXEC-NEXT: s_wait_dscnt 0x3
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index 181777a6d211d..76817ad0f39ca 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -733,9 +733,9 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[8:9]
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s0, 0xf000
; GCN-O0-NEXT: s_mov_b32 s2, 0
; GCN-O0-NEXT: s_mov_b32 s4, s2
@@ -745,9 +745,10 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: ; kill: def $sgpr0_sgpr1 killed $sgpr0_sgpr1 def $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b64 s[2:3], s[4:5]
; GCN-O0-NEXT: v_mov_b32_e32 v1, 1
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: buffer_store_dword v1, v[2:3], s[0:3], 0 addr64 offset:4
; GCN-O0-NEXT: s_mov_b32 s0, 2
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
; GCN-O0-NEXT: s_mov_b64 s[0:1], exec
; GCN-O0-NEXT: v_writelane_b32 v6, s0, 4
@@ -779,9 +780,9 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[8:9]
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s1, 0xf000
; GCN-O0-NEXT: s_mov_b32 s0, 0
; GCN-O0-NEXT: s_mov_b32 s2, s0
@@ -791,8 +792,9 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[2:3]
; GCN-O0-NEXT: v_mov_b32_e32 v1, 3
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: buffer_store_dword v1, v[2:3], s[4:7], 0 addr64 offset:12
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
; GCN-O0-NEXT: s_mov_b64 s[0:1], exec
; GCN-O0-NEXT: v_writelane_b32 v6, s0, 6
@@ -1326,15 +1328,12 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_readlane_b32 s5, v7, 19
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: ; %bb.11: ; %bb12
-; GCN-O0-NEXT: s_waitcnt expcnt(3)
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(2)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: v_mov_b32_e32 v4, v3
; GCN-O0-NEXT: ; implicit-def: $sgpr4
; GCN-O0-NEXT: v_mov_b32_e32 v5, s4
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 27d66a1886d0e..c59fbfd5837ce 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1877,28 +1877,28 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; VI-NEXT: v_add_u32_e32 v12, vcc, 6, v0
; VI-NEXT: v_addc_u32_e32 v13, vcc, 0, v1, vcc
; VI-NEXT: flat_load_ubyte v2, v[2:3]
+; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: flat_load_ubyte v3, v[10:11]
; VI-NEXT: flat_load_ubyte v4, v[4:5]
; VI-NEXT: flat_load_ubyte v5, v[6:7]
-; VI-NEXT: flat_load_ubyte v7, v[8:9]
-; VI-NEXT: flat_load_ubyte v3, v[10:11]
; VI-NEXT: flat_load_ubyte v6, v[12:13]
-; VI-NEXT: flat_load_ubyte v0, v[0:1]
+; VI-NEXT: flat_load_ubyte v7, v[8:9]
; VI-NEXT: v_mov_b32_e32 v8, 0x3020504
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: v_lshlrev_b32_e32 v9, 8, v2
; VI-NEXT: s_waitcnt vmcnt(5)
-; VI-NEXT: v_or_b32_e32 v4, v9, v4
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v5
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v3, v3
; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v7
+; VI-NEXT: v_or_b32_e32 v4, v9, v4
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v3, v3
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, v5
; VI-NEXT: v_perm_b32 v4, v4, s0, v8
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v7
; VI-NEXT: v_cvt_f32_ubyte0_e32 v6, v6
; VI-NEXT: v_cvt_f32_ubyte1_e32 v5, v4
; VI-NEXT: v_cvt_f32_ubyte0_e32 v4, v4
@@ -1914,18 +1914,19 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; GFX10-NEXT: v_mov_b32_e32 v7, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x5
-; GFX10-NEXT: global_load_ubyte v5, v0, s[2:3] offset:6
; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
; GFX10-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
; GFX10-NEXT: global_load_ubyte v6, v0, s[2:3] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v0, s[2:3] offset:6
; GFX10-NEXT: global_load_short_d16 v4, v0, s[2:3] offset:4
; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
-; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: s_waitcnt vmcnt(5)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
-; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: s_waitcnt vmcnt(4)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
-; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v6
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v6, v5
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v5, v4
@@ -1974,18 +1975,19 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x5
-; GFX11-NEXT: global_load_u8 v5, v0, s[2:3] offset:6
; GFX11-NEXT: global_load_u8 v1, v0, s[2:3] offset:3
; GFX11-NEXT: global_load_u8 v2, v0, s[2:3] offset:2
; GFX11-NEXT: global_load_u8 v6, v0, s[2:3] offset:1
+; GFX11-NEXT: global_load_u8 v5, v0, s[2:3] offset:6
; GFX11-NEXT: global_load_d16_b16 v4, v0, s[2:3] offset:4
; GFX11-NEXT: global_load_u8 v0, v0, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: s_waitcnt vmcnt(5)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
-; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, v6
+; GFX11-NEXT: s_waitcnt vmcnt(2)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v6, v5
; GFX11-NEXT: s_waitcnt vmcnt(1)
; GFX11-NEXT: v_cvt_f32_ubyte1_e32 v5, v4
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 5e7b30735a67a..cc24b7f12b721 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -555,47 +555,52 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_9
; GFX9-O0-NEXT: .LBB0_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 1
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
; GFX9-O0-NEXT: s_mov_b32 s4, 63
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8
; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
@@ -615,26 +620,29 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_4
; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while
@@ -645,35 +653,36 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v31, 9
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8
; GFX9-O0-NEXT: s_mov_b32 s5, 1
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
@@ -683,6 +692,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6
; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
@@ -692,41 +702,46 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_or_b32_e64 v7, v3, v6
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
; GFX9-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v28
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v26
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v11, v16
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v27
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v25
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v11
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v23
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v4
; GFX9-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13
; GFX9-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
@@ -743,13 +758,17 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v20
; GFX9-O0-NEXT: v_and_b32_e64 v16, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_and_b32_e64 v19, v11, v19
; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v22
; GFX9-O0-NEXT: v_and_b32_e64 v8, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v21
; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v16
; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
@@ -768,15 +787,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17
; GFX9-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18
; GFX9-O0-NEXT: v_mov_b32_e32 v11, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9
; GFX9-O0-NEXT: v_mov_b32_e32 v17, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v10, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
@@ -858,29 +881,30 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
@@ -924,17 +948,21 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
@@ -962,10 +990,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -973,40 +1002,44 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_6
; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s4, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3
; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
; GFX9-O0-NEXT: s_mov_b32 s6, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s9, s7
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
@@ -1033,10 +1066,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v12, s10, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v13
; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v14
@@ -1118,28 +1153,35 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execz .LBB0_5
; GFX9-O0-NEXT: s_branch .LBB0_7
; GFX9-O0-NEXT: .LBB0_9: ; %udiv-end
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-O0-NEXT: v_xor_b32_e64 v0, v0, v3
; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr1_vgpr2 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_xor_b32_e64 v5, v2, v1
; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v8
; GFX9-O0-NEXT: v_xor_b32_e64 v0, v0, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v9
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 killed $vgpr7_vgpr8 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_xor_b32_e64 v7, v7, v4
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v0
@@ -1392,10 +1434,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v6
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
@@ -1405,7 +1447,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s8, 0
; GFX9-G-O0-NEXT: v_writelane_b32 v31, s9, 1
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], 0x7f
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v1
@@ -1426,6 +1468,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v8, v0, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
@@ -1621,26 +1664,29 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 4
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 5
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_4
; GFX9-G-O0-NEXT: .LBB0_2: ; %Flow2
@@ -1665,18 +1711,18 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_8
; GFX9-G-O0-NEXT: .LBB0_3: ; %udiv-loop-exit
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
@@ -1690,8 +1736,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v5, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
@@ -1725,26 +1773,29 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_3
; GFX9-G-O0-NEXT: .LBB0_5: ; %udiv-do-while
@@ -1755,36 +1806,36 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-G-O0-NEXT: v_readlane_b32 s7, v31, 9
-; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 1
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
@@ -1799,6 +1850,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v0, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v25
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
@@ -1808,6 +1860,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v0, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v22
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v23
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v24
@@ -1822,10 +1875,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v15, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v27
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v28
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v29
; GFX9-G-O0-NEXT: v_mov_b32_e32 v24, v30
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
@@ -1850,10 +1903,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[10:11], v13, v4
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v12, v9, s[10:11]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v7, s[10:11]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[10:11], v6, v5, s[10:11]
; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
@@ -1878,9 +1934,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-G-O0-NEXT: v_and_b32_e64 v11, v10, v11
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_and_b32_e64 v10, v10, v21
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-G-O0-NEXT: v_and_b32_e64 v8, v6, v8
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_and_b32_e64 v6, v6, v20
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v4, s[8:9], v4, v11
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v9, v10, s[8:9]
@@ -1890,9 +1950,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v16
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
@@ -1953,32 +2017,34 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v17
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v16
; GFX9-G-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v4
; GFX9-G-O0-NEXT: s_mov_b32 s5, 0xffffffc0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s5
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v18, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4
; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v18
@@ -1987,7 +2053,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v18, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v18, v6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v18, v[20:21]
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[25:26], v18, v[22:23]
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v5, v[20:21]
@@ -2025,15 +2091,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b32 s7, -1
; GFX9-G-O0-NEXT: s_mov_b32 s6, -1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v16, s4
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v15, s[4:5], v15, v16
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, s10
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v14, s[4:5], v14, v15, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, s7
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v13, s[4:5], v13, v14, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, s6
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v12, s[4:5], v12, v13, s[4:5]
; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
@@ -2052,49 +2122,57 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_5
; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-bb1
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v2, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v7, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s8
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v1, v6, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v5
@@ -2110,8 +2188,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v8, s[6:7], v1, v2
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
; GFX9-G-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v9
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
@@ -2123,6 +2204,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[8:9], v8, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v8, v0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v8, v[12:13]
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[17:18], v14, v[12:13]
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[15:16], v8, v[10:11]
@@ -2194,27 +2276,31 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_cbranch_execz .LBB0_4
; GFX9-G-O0-NEXT: s_branch .LBB0_6
; GFX9-G-O0-NEXT: .LBB0_8: ; %udiv-end
-; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v9
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: v_xor_b32_e64 v0, v0, v7
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-G-O0-NEXT: v_xor_b32_e64 v1, v1, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-G-O0-NEXT: v_xor_b32_e64 v2, v2, v5
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_xor_b32_e64 v3, v3, v4
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v0, s[4:5], v0, v7
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v1, s[4:5], v1, v6, s[4:5]
@@ -2675,47 +2761,52 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_9
; GFX9-O0-NEXT: .LBB1_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 1
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
; GFX9-O0-NEXT: s_mov_b32 s4, 63
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8
; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
@@ -2735,26 +2826,29 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
@@ -2765,35 +2859,36 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v31, 9
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8
; GFX9-O0-NEXT: s_mov_b32 s5, 1
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
@@ -2803,6 +2898,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6
; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
@@ -2812,41 +2908,46 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_or_b32_e64 v7, v3, v6
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
; GFX9-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v28
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v26
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v11, v16
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v27
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v25
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v11
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v23
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v4
; GFX9-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13
; GFX9-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
@@ -2863,13 +2964,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v20
; GFX9-O0-NEXT: v_and_b32_e64 v16, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_and_b32_e64 v19, v11, v19
; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v22
; GFX9-O0-NEXT: v_and_b32_e64 v8, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v21
; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v16
; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
@@ -2888,15 +2993,19 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17
; GFX9-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18
; GFX9-O0-NEXT: v_mov_b32_e32 v11, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9
; GFX9-O0-NEXT: v_mov_b32_e32 v17, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v10, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
@@ -2978,29 +3087,30 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
@@ -3044,17 +3154,21 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
@@ -3082,10 +3196,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -3093,40 +3208,44 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s4, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3
; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
; GFX9-O0-NEXT: s_mov_b32 s6, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s9, s7
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
@@ -3153,10 +3272,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v12, s10, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v13
; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v14
@@ -3238,14 +3359,15 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_5
; GFX9-O0-NEXT: s_branch .LBB1_7
; GFX9-O0-NEXT: .LBB1_9: ; %udiv-end
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 32
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[2:3], s4, v[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6
@@ -3652,26 +3774,29 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v32, 4
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v32, 5
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_4
; GFX9-G-O0-NEXT: .LBB1_2: ; %Flow2
@@ -3696,18 +3821,18 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_8
; GFX9-G-O0-NEXT: .LBB1_3: ; %udiv-loop-exit
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-G-O0-NEXT: s_mov_b32 s4, 1
@@ -3721,8 +3846,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v5, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
@@ -3756,26 +3883,29 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v32, 6
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v32, 7
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_3
; GFX9-G-O0-NEXT: .LBB1_5: ; %udiv-do-while
@@ -3786,36 +3916,36 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_readlane_b32 s6, v32, 8
; GFX9-G-O0-NEXT: v_readlane_b32 s7, v32, 9
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[8:9], 1
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v4
; GFX9-G-O0-NEXT: s_mov_b32 s10, 1
@@ -3830,6 +3960,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 killed $vgpr4_vgpr5 killed $exec
; GFX9-G-O0-NEXT: v_or_b32_e64 v7, v0, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v15
; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
@@ -3839,6 +3970,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v3
; GFX9-G-O0-NEXT: v_or_b32_e64 v4, v0, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v14
@@ -3853,10 +3985,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_lshrrev_b32_e64 v13, v2, v3
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v28
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v29
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v30
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v1
@@ -3881,10 +4013,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[10:11], v11, v4
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v10, v9, s[10:11]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[10:11], v8, v7, s[10:11]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[10:11], v6, v5, s[10:11]
; GFX9-G-O0-NEXT: s_mov_b32 s10, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s10
@@ -3909,8 +4044,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v24
; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v25
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v26
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v27
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v22
@@ -3929,9 +4066,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v8
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v16
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v17
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v18
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v19
; GFX9-G-O0-NEXT: s_mov_b32 s8, -1
; GFX9-G-O0-NEXT: s_mov_b32 s12, -1
@@ -3992,32 +4133,33 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v7
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v6
; GFX9-G-O0-NEXT: s_mov_b32 s5, 0xffffffc0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s5
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-G-O0-NEXT: v_add_u32_e64 v4, v12, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s4
; GFX9-G-O0-NEXT: v_sub_u32_e64 v5, v5, v12
@@ -4060,9 +4202,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: ; kill: def $vgpr4_vgpr5 killed $vgpr4_vgpr5 def $vgpr4_vgpr5_vgpr6_vgpr7 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, v13
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v16
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v17
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v18
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v19
; GFX9-G-O0-NEXT: s_mov_b32 s4, -1
; GFX9-G-O0-NEXT: s_mov_b32 s10, -1
@@ -4096,49 +4242,57 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_5
; GFX9-G-O0-NEXT: .LBB1_7: ; %udiv-bb1
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v1, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v2, v3, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v6, s[6:7], v0, v2, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v5
@@ -4153,6 +4307,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s6
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v3, s[6:7], v0, v1
; GFX9-G-O0-NEXT: s_mov_b32 s7, 64
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v13, v9
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v8
; GFX9-G-O0-NEXT: s_mov_b32 s6, 0xffffffc0
@@ -4167,6 +4322,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v3, v0
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[0:1], v3, v[12:13]
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[17:18], v8, v[12:13]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[15:16], v3, v[10:11]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v18
diff --git a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
index 16a998bb5e9bf..47ae9afaee524 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
@@ -237,21 +237,24 @@ define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4
+; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5
; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0
; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:1
; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:2
-; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4
-; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5
; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:3
; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:7
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v5 offset:4
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v6 offset:5
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v2
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v3 offset:1
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v4 offset:2
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v7 offset:3
@@ -449,30 +452,38 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT: ds_read_u8 v9, v0 offset:8
+; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9
+; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4
+; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5
; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v0
; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0 offset:1
; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2
; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:3
-; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4
-; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5
; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:7
-; ALIGNED-SDAG-NEXT: ds_read_u8 v9, v0 offset:8
-; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9
; ALIGNED-SDAG-NEXT: ds_read_u8 v11, v0 offset:10
; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:11
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v12, s1
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v9 offset:8
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v10 offset:9
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v5 offset:4
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v6 offset:5
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v1
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v2 offset:1
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v3 offset:2
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v4 offset:3
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v7 offset:6
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v8 offset:7
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)
; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v11 offset:10
@@ -596,17 +607,18 @@ define amdgpu_kernel void @ds12align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0
; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:8
; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4
+; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0
; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:2
; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:10
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:8
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:2
@@ -799,37 +811,42 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT: ds_read_u8 v13, v0 offset:12
+; ALIGNED-SDAG-NEXT: ds_read_u8 v14, v0 offset:13
; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v0
; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0 offset:1
-; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2
-; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:3
; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4
; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5
-; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:6
-; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:7
; ALIGNED-SDAG-NEXT: ds_read_u8 v9, v0 offset:8
; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9
+; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2
+; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:3
+; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:6
+; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:7
; ALIGNED-SDAG-NEXT: ds_read_u8 v11, v0 offset:10
; ALIGNED-SDAG-NEXT: ds_read_u8 v12, v0 offset:11
-; ALIGNED-SDAG-NEXT: ds_read_u8 v13, v0 offset:12
-; ALIGNED-SDAG-NEXT: ds_read_u8 v14, v0 offset:13
; ALIGNED-SDAG-NEXT: ds_read_u8 v15, v0 offset:14
; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:15
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v16, s1
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v13 offset:12
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v14 offset:13
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v1
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v2 offset:1
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v5 offset:4
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v6 offset:5
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v9 offset:8
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v10 offset:9
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v3 offset:2
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v4 offset:3
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v7 offset:6
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v8 offset:7
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v11 offset:10
; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v12 offset:11
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)
@@ -982,22 +999,26 @@ define amdgpu_kernel void @ds16align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:12
; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0
-; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:2
; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4
-; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:12
-; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u16 v7, v0 offset:8
+; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:2
+; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6
; ALIGNED-SDAG-NEXT: ds_read_u16 v8, v0 offset:10
; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:14
; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:12
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4
-; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v7 offset:8
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:2
+; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:6
; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v8 offset:10
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 16b451699a9b1..6445ad7018289 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -530,19 +530,20 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:1
-; CI-NEXT: ds_read_u8 v3, v1 offset:34
-; CI-NEXT: ds_read_u8 v4, v1 offset:32
-; CI-NEXT: ds_read_u8 v5, v1 offset:2
; CI-NEXT: ds_read_u8 v6, v1
; CI-NEXT: ds_read_u8 v7, v1 offset:3
+; CI-NEXT: ds_read_u8 v5, v1 offset:2
; CI-NEXT: ds_read_u8 v8, v1 offset:33
+; CI-NEXT: ds_read_u8 v4, v1 offset:32
+; CI-NEXT: ds_read_u8 v3, v1 offset:34
; CI-NEXT: ds_read_u8 v1, v1 offset:35
; CI-NEXT: s_waitcnt lgkmcnt(7)
; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; CI-NEXT: s_waitcnt lgkmcnt(3)
+; CI-NEXT: s_waitcnt lgkmcnt(6)
; CI-NEXT: v_or_b32_e32 v2, v2, v6
-; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: s_waitcnt lgkmcnt(5)
; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v7
+; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_or_b32_e32 v5, v6, v5
; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -623,19 +624,20 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:6
-; CI-NEXT: ds_read_u8 v3, v1 offset:11
-; CI-NEXT: ds_read_u8 v4, v1 offset:9
-; CI-NEXT: ds_read_u8 v5, v1 offset:7
; CI-NEXT: ds_read_u8 v6, v1 offset:5
; CI-NEXT: ds_read_u8 v7, v1 offset:8
+; CI-NEXT: ds_read_u8 v5, v1 offset:7
; CI-NEXT: ds_read_u8 v8, v1 offset:10
+; CI-NEXT: ds_read_u8 v4, v1 offset:9
+; CI-NEXT: ds_read_u8 v3, v1 offset:11
; CI-NEXT: ds_read_u8 v1, v1 offset:12
; CI-NEXT: s_waitcnt lgkmcnt(7)
; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; CI-NEXT: s_waitcnt lgkmcnt(3)
+; CI-NEXT: s_waitcnt lgkmcnt(6)
; CI-NEXT: v_or_b32_e32 v2, v2, v6
-; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: s_waitcnt lgkmcnt(5)
; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v7
+; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_or_b32_e32 v5, v6, v5
; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -716,8 +718,8 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
; CI-NEXT: ds_read_u16 v2, v1 offset:2
-; CI-NEXT: ds_read_u16 v3, v1 offset:32
; CI-NEXT: ds_read_u16 v4, v1
+; CI-NEXT: ds_read_u16 v3, v1 offset:32
; CI-NEXT: ds_read_u16 v1, v1 offset:34
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -1454,21 +1456,21 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
; CI-NEXT: v_mov_b32_e32 v0, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_read_u8 v1, v0 offset:70
+; CI-NEXT: ds_read_u8 v4, v0 offset:69
; CI-NEXT: ds_read_u8 v2, v0 offset:72
; CI-NEXT: ds_read_u8 v3, v0 offset:71
-; CI-NEXT: ds_read_u8 v4, v0 offset:69
; CI-NEXT: ds_read_u8 v5, v0 offset:68
; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; CI-NEXT: s_waitcnt lgkmcnt(3)
-; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; CI-NEXT: s_waitcnt lgkmcnt(2)
-; CI-NEXT: v_or_b32_e32 v2, v2, v3
-; CI-NEXT: s_waitcnt lgkmcnt(1)
; CI-NEXT: v_or_b32_e32 v1, v1, v4
; CI-NEXT: ds_read_u8 v4, v0 offset:66
; CI-NEXT: ds_read_u8 v6, v0 offset:67
; CI-NEXT: ds_read_u8 v0, v0 offset:65
+; CI-NEXT: s_waitcnt lgkmcnt(5)
+; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT: s_waitcnt lgkmcnt(4)
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: v_or_b32_e32 v1, v2, v1
@@ -1487,14 +1489,14 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
; GFX9-ALIGNED-LABEL: read2_v2i32_align1_odd_offset:
; GFX9-ALIGNED: ; %bb.0: ; %entry
; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v2 offset:65
-; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v2 offset:66
-; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v2 offset:67
-; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v2 offset:68
; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v2 offset:70
; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v2 offset:69
; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v2 offset:72
; GFX9-ALIGNED-NEXT: ds_read_u8 v8, v2 offset:71
+; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v2 offset:66
+; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v2 offset:65
+; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v2 offset:68
+; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v2 offset:67
; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v1, 8, v1
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index aabed8e2814c0..af3115795d24d 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -2969,6 +2969,7 @@ define float @float32_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v63, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v63, 3
+; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
@@ -3001,12 +3002,12 @@ define float @float32_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v32
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v32
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -3381,6 +3382,7 @@ define double @double16_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v34, 35
; GCN-O0-NEXT: v_readlane_b32 s5, v34, 36
+; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -3413,12 +3415,12 @@ define double @double16_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v32
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v32
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -3553,6 +3555,7 @@ define double @double16_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v34, 39
; GCN-O0-NEXT: v_readlane_b32 s5, v34, 40
+; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
@@ -3585,12 +3588,12 @@ define double @double16_extelt_vec(i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v32
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v32
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -3693,15 +3696,16 @@ define i32 @extract_dyn_i32_3(<3 x i32> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v4, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v4, 3
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v3
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v3
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -3805,15 +3809,16 @@ define float @extract_dyn_float_3(<3 x float> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v4, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v4, 3
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v3
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v3
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -3933,17 +3938,18 @@ define i32 @extract_dyn_i32_5(<5 x i32> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 3
+; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v5
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v5
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -4067,17 +4073,18 @@ define float @extract_dyn_float_5(<5 x float> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 3
+; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v5
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v5
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -4215,18 +4222,19 @@ define i32 @extract_dyn_i32_6(<6 x i32> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v7, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v7, 3
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v6
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -4362,18 +4370,19 @@ define float @extract_dyn_float_6(<6 x float> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v7, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v7, 3
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v6
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -4525,6 +4534,7 @@ define i32 @extract_dyn_i32_7(<7 x i32> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v8, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v8, 3
+; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -4532,12 +4542,12 @@ define i32 @extract_dyn_i32_7(<7 x i32> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v7
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v7
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
@@ -4681,6 +4691,7 @@ define float @extract_dyn_float_7(<7 x float> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v8, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v8, 3
+; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -4688,12 +4699,12 @@ define float @extract_dyn_float_7(<7 x float> inreg %arg, i32 %idx) {
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v7
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v7
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movrels_b32_e32 v0, v0
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
index b42690eb3bf2b..ff9815858e618 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
@@ -154,15 +154,15 @@ define i32 @global_load_2xi16_align1(ptr addrspace(1) %p) #0 {
; GFX7-ALIGNED-NEXT: v_add_i32_e32 v6, vcc, 3, v0
; GFX7-ALIGNED-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
; GFX7-ALIGNED-NEXT: flat_load_ubyte v4, v[4:5]
-; GFX7-ALIGNED-NEXT: flat_load_ubyte v5, v[6:7]
; GFX7-ALIGNED-NEXT: flat_load_ubyte v2, v[2:3]
+; GFX7-ALIGNED-NEXT: flat_load_ubyte v5, v[6:7]
; GFX7-ALIGNED-NEXT: flat_load_ubyte v0, v[0:1]
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(3)
; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v1, 8, v4
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 24, v5
-; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(1)
; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(1)
+; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 24, v5
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(0)
; GFX7-ALIGNED-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-ALIGNED-NEXT: v_or_b32_e32 v1, v3, v2
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
index 345cef60501b8..1361fbc4f967a 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
@@ -205,15 +205,15 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
; GFX7-ALIGNED-NEXT: v_add_i32_e32 v2, vcc, 1, v0
; GFX7-ALIGNED-NEXT: v_add_i32_e32 v3, vcc, 3, v0
; GFX7-ALIGNED-NEXT: buffer_load_ubyte v2, v2, s[0:3], 0 offen
-; GFX7-ALIGNED-NEXT: buffer_load_ubyte v3, v3, s[0:3], 0 offen
; GFX7-ALIGNED-NEXT: buffer_load_ubyte v1, v1, s[0:3], 0 offen
+; GFX7-ALIGNED-NEXT: buffer_load_ubyte v3, v3, s[0:3], 0 offen
; GFX7-ALIGNED-NEXT: buffer_load_ubyte v0, v0, s[0:3], 0 offen
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(3)
; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(1)
; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(1)
+; GFX7-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; GFX7-ALIGNED-NEXT: s_waitcnt vmcnt(0)
; GFX7-ALIGNED-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7-ALIGNED-NEXT: v_or_b32_e32 v1, v3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 56415ef44c527..b82823a041dc2 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -497,13 +497,13 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e64 v6, s[4:5], 28, v0
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
@@ -535,13 +535,13 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e64 v6, s[0:1], 28, v0
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 433dc96366e62..602ebd4bedfb7 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -2361,8 +2361,8 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
; GFX8-GFX803-SDAG-NEXT: flat_load_dword v18, v[14:15]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v16, vcc, 56, v2
@@ -2376,8 +2376,9 @@ define void @freeze_v15i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[16:17], v18
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[8:11]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(4)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[6:7], v[0:1]
@@ -3433,19 +3434,21 @@ define void @freeze_v19i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
-; GFX6-GISEL-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[20:21], v[0:1], s[4:7], 0 addr64 offset:64
+; GFX6-GISEL-NEXT: buffer_load_dword v22, v[0:1], s[4:7], 0 addr64 offset:72
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(3)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(3)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[20:21], v[2:3], s[4:7], 0 addr64 offset:64
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(5)
; GFX6-GISEL-NEXT: buffer_store_dword v22, v[2:3], s[4:7], 0 addr64 offset:72
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -5205,28 +5208,30 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX6-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX6-GISEL-NEXT: buffer_load_dword v34, v[0:1], s[4:7], 0 addr64 offset:120
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:32
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[4:7], 0 addr64 offset:48
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[4:7], 0 addr64 offset:64
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[4:7], 0 addr64 offset:80
-; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
; GFX6-GISEL-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[4:7], 0 addr64 offset:96
+; GFX6-GISEL-NEXT: buffer_load_dwordx2 v[32:33], v[0:1], s[4:7], 0 addr64 offset:112
+; GFX6-GISEL-NEXT: buffer_load_dword v34, v[0:1], s[4:7], 0 addr64 offset:120
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v[2:3], s[4:7], 0 addr64
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[8:11], v[2:3], s[4:7], 0 addr64 offset:16
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(7)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[12:15], v[2:3], s[4:7], 0 addr64 offset:32
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(7)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[16:19], v[2:3], s[4:7], 0 addr64 offset:48
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(7)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[20:23], v[2:3], s[4:7], 0 addr64 offset:64
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(7)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[24:27], v[2:3], s[4:7], 0 addr64 offset:80
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(6)
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx4 v[28:31], v[2:3], s[4:7], 0 addr64 offset:96
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dwordx2 v[32:33], v[2:3], s[4:7], 0 addr64 offset:112
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(8)
; GFX6-GISEL-NEXT: buffer_store_dword v34, v[2:3], s[4:7], 0 addr64 offset:120
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -5325,9 +5330,9 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v15, vcc, 0, v1, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v20, vcc, 32, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dword v34, v[14:15]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[0:1]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v0
@@ -5351,8 +5356,9 @@ define void @freeze_v31i32(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v10, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v11, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[16:19]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[4:5], v[12:15]
; GFX8-GFX803-SDAG-NEXT: flat_store_dword v[6:7], v34
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[32:33]
@@ -10293,8 +10299,8 @@ define void @freeze_v16p0(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x60
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v28, vcc, s6, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
@@ -10304,28 +10310,29 @@ define void @freeze_v16p0(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s5, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35]
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -11518,8 +11525,8 @@ define void @freeze_v16p1(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: s_movk_i32 s6, 0x60
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v28, vcc, s6, v0
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[0:1]
+; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
; GFX8-GFX803-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
@@ -11529,28 +11536,29 @@ define void @freeze_v16p1(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s6, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
+; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35]
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s5, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 64, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, s4, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 32, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 48, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(5)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(6)
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 16, v2
; GFX8-GFX803-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[32:35]
; GFX8-GFX803-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -12086,18 +12094,18 @@ define void @freeze_v8p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 24, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 16, v0
; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
-; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v0
+; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
+; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v0
; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 16, v1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[4:5]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 24, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[2:3]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-SDAG-NEXT: ds_write_b64 v1, v[6:7]
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[8:9]
@@ -12269,31 +12277,35 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v16, vcc, 56, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v14, vcc, 48, v0
; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
-; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
-; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v6
; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v0
-; GFX6-SDAG-NEXT: ds_read_b64 v[10:11], v10
-; GFX6-SDAG-NEXT: ds_read_b64 v[12:13], v12
; GFX6-SDAG-NEXT: ds_read_b64 v[14:15], v14
; GFX6-SDAG-NEXT: ds_read_b64 v[16:17], v16
+; GFX6-SDAG-NEXT: ds_read_b64 v[12:13], v12
+; GFX6-SDAG-NEXT: ds_read_b64 v[10:11], v10
+; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v6
+; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
+; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 48, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(4)
-; GFX6-SDAG-NEXT: ds_write_b64 v1, v[8:9]
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[14:15]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 56, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[16:17]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 32, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[12:13]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 40, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[10:11]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 16, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[6:7]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 24, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[4:5]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v1
+; GFX6-SDAG-NEXT: ds_write_b64 v1, v[8:9]
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(7)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[2:3]
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -12716,14 +12728,15 @@ define void @freeze_v3p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v0
; GFX6-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 4, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v5, vcc, 8, v1
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX6-SDAG-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
@@ -12753,14 +12766,15 @@ define void @freeze_v3p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG: ; %bb.0:
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v0
; GFX7-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, 4, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, 8, v1
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX7-SDAG-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX7-SDAG-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX7-SDAG-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
@@ -12790,14 +12804,15 @@ define void @freeze_v3p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG: ; %bb.0:
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 4, v0
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 8, v0
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 4, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v5, vcc, 8, v1
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
@@ -12872,17 +12887,19 @@ define void @freeze_v4p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 8, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v3, vcc, 4, v0
-; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 12, v0
; GFX6-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v5, vcc, 4, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v6, vcc, 8, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX6-SDAG-NEXT: buffer_store_dword v0, v7, s[0:3], 0 offen
@@ -12918,17 +12935,19 @@ define void @freeze_v4p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, 8, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, 4, v0
-; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, 12, v0
; GFX7-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, 4, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, 8, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX7-SDAG-NEXT: buffer_store_dword v0, v7, s[0:3], 0 offen
@@ -12964,17 +12983,19 @@ define void @freeze_v4p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 8, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v3, vcc, 4, v0
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 12, v0
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v5, vcc, 4, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 8, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v7, vcc, 12, v1
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(3)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v0, v7, s[0:3], 0 offen
@@ -13064,13 +13085,13 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v5, vcc, 12, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v6, vcc, 8, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v7, vcc, 4, v0
-; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 28, v0
; GFX6-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v9, vcc, 4, v1
@@ -13080,13 +13101,19 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v13, vcc, 20, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v14, vcc, 24, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v15, vcc, 28, v1
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v7, v9, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v6, v10, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v5, v11, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v4, v12, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v3, v13, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v2, v14, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX6-SDAG-NEXT: buffer_store_dword v0, v15, s[0:3], 0 offen
@@ -13146,13 +13173,13 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, 12, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, 8, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, 4, v0
-; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, 28, v0
; GFX7-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v9, vcc, 4, v1
@@ -13162,13 +13189,19 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG-NEXT: v_add_i32_e32 v13, vcc, 20, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v14, vcc, 24, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v15, vcc, 28, v1
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v7, v9, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v6, v10, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v5, v11, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v4, v12, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v3, v13, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v2, v14, s[0:3], 0 offen
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX7-SDAG-NEXT: buffer_store_dword v0, v15, s[0:3], 0 offen
@@ -13228,13 +13261,13 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v5, vcc, 12, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 8, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v7, vcc, 4, v0
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 28, v0
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v9, vcc, 4, v1
@@ -13244,13 +13277,19 @@ define void @freeze_v8p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v13, vcc, 20, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v14, vcc, 24, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v15, vcc, 28, v1
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v7, v9, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v6, v10, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v5, v11, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v4, v12, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v3, v13, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v2, v14, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(7)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v0, v15, s[0:3], 0 offen
@@ -13398,10 +13437,10 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v6, vcc, 12, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v7, vcc, 8, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v8, vcc, 4, v0
-; GFX6-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 56, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v3, vcc, 52, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 48, v0
@@ -13412,53 +13451,54 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: v_add_i32_e32 v13, vcc, 28, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v14, vcc, 24, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v15, vcc, 20, v0
-; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v15, v15, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v14, v14, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v13, v13, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_load_dword v12, v12, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 60, v0
; GFX6-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v17, vcc, 4, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v18, vcc, 8, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v19, vcc, 12, v1
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX6-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX6-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v8, v17, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v8, vcc, 16, v1
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(13)
; GFX6-SDAG-NEXT: buffer_store_dword v5, v8, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v5, vcc, 40, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v17, vcc, 20, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v7, vcc, 24, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v18, vcc, 28, v1
+; GFX6-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v6, vcc, 32, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v19, vcc, 36, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v8, vcc, 44, v1
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX6-SDAG-NEXT: buffer_store_dword v15, v17, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v14, v7, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX6-SDAG-NEXT: buffer_store_dword v13, v18, s[0:3], 0 offen
-; GFX6-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v12, v6, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v10, v5, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
; GFX6-SDAG-NEXT: v_add_i32_e32 v5, vcc, 48, v1
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX6-SDAG-NEXT: buffer_store_dword v4, v5, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 52, v1
@@ -13466,8 +13506,8 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-SDAG-NEXT: s_waitcnt expcnt(0)
; GFX6-SDAG-NEXT: v_add_i32_e32 v3, vcc, 56, v1
; GFX6-SDAG-NEXT: v_add_i32_e32 v1, vcc, 60, v1
-; GFX6-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(14)
+; GFX6-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX6-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -13489,8 +13529,8 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-GISEL-NEXT: v_add_i32_e32 v12, vcc, 40, v0
; GFX6-GISEL-NEXT: v_add_i32_e32 v13, vcc, 44, v0
; GFX6-GISEL-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX6-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX6-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
@@ -13520,35 +13560,34 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX6-GISEL-NEXT: v_add_i32_e32 v18, vcc, 28, v1
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX6-GISEL-NEXT: buffer_store_dword v4, v19, s[0:3], 0 offen
-; GFX6-GISEL-NEXT: s_waitcnt expcnt(0)
-; GFX6-GISEL-NEXT: v_add_i32_e32 v4, vcc, 32, v1
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX6-GISEL-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 40, v1
+; GFX6-GISEL-NEXT: s_waitcnt expcnt(1)
+; GFX6-GISEL-NEXT: v_add_i32_e32 v4, vcc, 32, v1
; GFX6-GISEL-NEXT: v_add_i32_e32 v19, vcc, 36, v1
; GFX6-GISEL-NEXT: s_waitcnt expcnt(0)
; GFX6-GISEL-NEXT: v_add_i32_e32 v5, vcc, 44, v1
-; GFX6-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_store_dword v6, v17, s[0:3], 0 offen
-; GFX6-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_store_dword v8, v18, s[0:3], 0 offen
-; GFX6-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
-; GFX6-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_store_dword v13, v5, s[0:3], 0 offen
; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 48, v1
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX6-GISEL-NEXT: buffer_store_dword v14, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 52, v1
-; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX6-GISEL-NEXT: buffer_store_dword v15, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: v_add_i32_e32 v2, vcc, 56, v1
; GFX6-GISEL-NEXT: v_add_i32_e32 v1, vcc, 60, v1
-; GFX6-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX6-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX6-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -13560,10 +13599,10 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, 12, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, 8, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v8, vcc, 4, v0
-; GFX7-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, 56, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, 52, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, 48, v0
@@ -13574,58 +13613,57 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-SDAG-NEXT: v_add_i32_e32 v13, vcc, 28, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v14, vcc, 24, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v15, vcc, 20, v0
-; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v15, v15, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v14, v14, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v13, v13, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_load_dword v12, v12, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, 60, v0
; GFX7-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v17, vcc, 4, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v18, vcc, 8, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v19, vcc, 12, v1
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX7-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX7-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v8, v17, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v8, vcc, 16, v1
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(13)
; GFX7-SDAG-NEXT: buffer_store_dword v5, v8, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, 40, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v17, vcc, 20, v1
+; GFX7-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, 24, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v18, vcc, 28, v1
+; GFX7-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, 32, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v19, vcc, 36, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v8, vcc, 44, v1
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX7-SDAG-NEXT: buffer_store_dword v15, v17, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v14, v7, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX7-SDAG-NEXT: buffer_store_dword v13, v18, s[0:3], 0 offen
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v12, v6, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v10, v5, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, 48, v1
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX7-SDAG-NEXT: buffer_store_dword v4, v5, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, 52, v1
; GFX7-SDAG-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, 56, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, 60, v1
-; GFX7-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(14)
+; GFX7-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX7-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -13647,8 +13685,8 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-GISEL-NEXT: v_add_i32_e32 v12, vcc, 40, v0
; GFX7-GISEL-NEXT: v_add_i32_e32 v13, vcc, 44, v0
; GFX7-GISEL-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX7-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX7-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
@@ -13676,33 +13714,32 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX7-GISEL-NEXT: v_add_i32_e32 v18, vcc, 28, v1
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX7-GISEL-NEXT: buffer_store_dword v4, v19, s[0:3], 0 offen
-; GFX7-GISEL-NEXT: v_add_i32_e32 v4, vcc, 32, v1
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX7-GISEL-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 40, v1
+; GFX7-GISEL-NEXT: v_add_i32_e32 v4, vcc, 32, v1
; GFX7-GISEL-NEXT: v_add_i32_e32 v19, vcc, 36, v1
; GFX7-GISEL-NEXT: v_add_i32_e32 v5, vcc, 44, v1
-; GFX7-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_store_dword v6, v17, s[0:3], 0 offen
-; GFX7-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_store_dword v8, v18, s[0:3], 0 offen
-; GFX7-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
-; GFX7-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_store_dword v13, v5, s[0:3], 0 offen
; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 48, v1
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX7-GISEL-NEXT: buffer_store_dword v14, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 52, v1
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX7-GISEL-NEXT: buffer_store_dword v15, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: v_add_i32_e32 v2, vcc, 56, v1
; GFX7-GISEL-NEXT: v_add_i32_e32 v1, vcc, 60, v1
-; GFX7-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX7-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX7-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -13714,10 +13751,10 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 12, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v7, vcc, 8, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 4, v0
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v2, vcc, 56, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v3, vcc, 52, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 48, v0
@@ -13728,58 +13765,57 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v13, vcc, 28, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v14, vcc, 24, v0
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v15, vcc, 20, v0
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v15, v15, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v14, v14, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v13, v13, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v12, v12, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v11, v11, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v10, v10, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v3, v3, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v0, vcc, 60, v0
; GFX8-GFX803-SDAG-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v17, vcc, 4, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 8, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v19, vcc, 12, v1
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
-; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v8, v17, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 16, v1
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(13)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v5, v8, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v5, vcc, 40, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v17, vcc, 20, v1
+; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v7, v18, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v7, vcc, 24, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v18, vcc, 28, v1
+; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v6, v19, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v6, vcc, 32, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v19, vcc, 36, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v8, vcc, 44, v1
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v15, v17, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v14, v7, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(9)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v13, v18, s[0:3], 0 offen
-; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(9)
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v12, v6, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v10, v5, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v5, vcc, 48, v1
+; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v4, v5, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v4, vcc, 52, v1
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v3, vcc, 56, v1
; GFX8-GFX803-SDAG-NEXT: v_add_u32_e32 v1, vcc, 60, v1
-; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -13801,8 +13837,8 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GISEL-NEXT: v_add_u32_e32 v12, vcc, 40, v0
; GFX8-GISEL-NEXT: v_add_u32_e32 v13, vcc, 44, v0
; GFX8-GISEL-NEXT: buffer_load_dword v4, v4, s[0:3], 0 offen
-; GFX8-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_load_dword v5, v5, s[0:3], 0 offen
+; GFX8-GISEL-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_load_dword v6, v6, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_load_dword v7, v7, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
@@ -13830,33 +13866,32 @@ define void @freeze_v16p5(ptr addrspace(5) %ptra, ptr addrspace(5) %ptrb) {
; GFX8-GISEL-NEXT: v_add_u32_e32 v18, vcc, 28, v1
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX8-GISEL-NEXT: buffer_store_dword v4, v19, s[0:3], 0 offen
-; GFX8-GISEL-NEXT: v_add_u32_e32 v4, vcc, 32, v1
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX8-GISEL-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: v_add_u32_e32 v2, vcc, 40, v1
+; GFX8-GISEL-NEXT: v_add_u32_e32 v4, vcc, 32, v1
; GFX8-GISEL-NEXT: v_add_u32_e32 v19, vcc, 36, v1
; GFX8-GISEL-NEXT: v_add_u32_e32 v5, vcc, 44, v1
-; GFX8-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_store_dword v6, v17, s[0:3], 0 offen
-; GFX8-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT: buffer_store_dword v7, v3, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_store_dword v8, v18, s[0:3], 0 offen
-; GFX8-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT: buffer_store_dword v10, v4, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_store_dword v11, v19, s[0:3], 0 offen
-; GFX8-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT: buffer_store_dword v12, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_store_dword v13, v5, s[0:3], 0 offen
; GFX8-GISEL-NEXT: v_add_u32_e32 v2, vcc, 48, v1
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX8-GISEL-NEXT: buffer_store_dword v14, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: v_add_u32_e32 v2, vcc, 52, v1
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
; GFX8-GISEL-NEXT: buffer_store_dword v15, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: v_add_u32_e32 v2, vcc, 56, v1
; GFX8-GISEL-NEXT: v_add_u32_e32 v1, vcc, 60, v1
-; GFX8-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(14)
+; GFX8-GISEL-NEXT: buffer_store_dword v16, v2, s[0:3], 0 offen
; GFX8-GISEL-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 4134420e78a2e..c0ee3aad7f265 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1233,6 +1233,7 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s0, s8
; SI-NEXT: s_mov_b32 s1, s9
@@ -1240,13 +1241,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; SI-NEXT: s_mov_b32 s9, s11
; SI-NEXT: s_mov_b32 s10, s2
; SI-NEXT: s_mov_b32 s11, s3
-; SI-NEXT: s_mov_b32 s7, s3
-; SI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; SI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
+; SI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
; SI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
; SI-NEXT: v_rcp_f32_e32 v3, v1
; SI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
@@ -1322,6 +1322,7 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: s_mov_b32 s6, s2
+; CI-NEXT: s_mov_b32 s7, s3
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_mov_b32 s0, s8
; CI-NEXT: s_mov_b32 s1, s9
@@ -1329,13 +1330,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; CI-NEXT: s_mov_b32 s9, s11
; CI-NEXT: s_mov_b32 s10, s2
; CI-NEXT: s_mov_b32 s11, s3
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; CI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
+; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
; CI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
; CI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
; CI-NEXT: v_rcp_f32_e32 v3, v1
@@ -1444,11 +1444,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-NEXT: global_load_ushort v2, v0, s[6:7] offset:8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX9-NEXT: v_mad_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
; GFX9-NEXT: v_mac_f32_e32 v4, v5, v3
@@ -1471,12 +1472,12 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX10-NEXT: global_load_ushort v2, v0, s[6:7] offset:8
+; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX10-NEXT: v_rcp_f32_e32 v4, v3
; GFX10-NEXT: v_fma_mix_f32 v5, v1, v4, neg(0) op_sel_hi:[1,0,0]
; GFX10-NEXT: v_mad_f32 v7, -v3, v5, v6
@@ -1500,12 +1501,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX11-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX11-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -1534,12 +1536,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX11-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX11-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
@@ -1568,12 +1571,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1150-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-TRUE16-NEXT: s_clause 0x1
-; GFX1150-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX1150-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX1150-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1150-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -1603,12 +1607,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1150-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1150-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-FAKE16-NEXT: s_clause 0x1
-; GFX1150-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX1150-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX1150-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1150-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
@@ -1638,12 +1643,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x1
; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -1673,12 +1679,13 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX1200-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX1200-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1200-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
@@ -1716,6 +1723,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_mov_b32 s7, s3
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s0, s8
; SI-NEXT: s_mov_b32 s1, s9
@@ -1723,13 +1731,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: s_mov_b32 s9, s11
; SI-NEXT: s_mov_b32 s10, s2
; SI-NEXT: s_mov_b32 s11, s3
-; SI-NEXT: s_mov_b32 s7, s3
-; SI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; SI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
+; SI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
; SI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
; SI-NEXT: v_rcp_f32_e32 v3, v1
; SI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
@@ -1805,6 +1812,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: s_mov_b32 s6, s2
+; CI-NEXT: s_mov_b32 s7, s3
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_mov_b32 s0, s8
; CI-NEXT: s_mov_b32 s1, s9
@@ -1812,13 +1820,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: s_mov_b32 s9, s11
; CI-NEXT: s_mov_b32 s10, s2
; CI-NEXT: s_mov_b32 s11, s3
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; CI-NEXT: buffer_load_ushort v1, off, s[4:7], 0 offset:8
+; CI-NEXT: buffer_load_ushort v0, off, s[8:11], 0
; CI-NEXT: s_waitcnt vmcnt(1)
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; CI-NEXT: s_waitcnt vmcnt(0)
+; CI-NEXT: v_cvt_f32_f16_e32 v2, v0
; CI-NEXT: v_div_scale_f32 v1, s[4:5], v4, v4, v2
; CI-NEXT: v_div_scale_f32 v0, vcc, v2, v4, v2
; CI-NEXT: v_rcp_f32_e32 v3, v1
@@ -1927,11 +1934,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX9-NEXT: global_load_ushort v2, v0, s[6:7] offset:8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v2
; GFX9-NEXT: v_rcp_f32_e32 v3, v3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mad_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX9-NEXT: v_mad_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
; GFX9-NEXT: v_mac_f32_e32 v4, v5, v3
@@ -1954,12 +1962,12 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX10-NEXT: global_load_ushort v2, v0, s[6:7] offset:8
+; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v1
; GFX10-NEXT: v_rcp_f32_e32 v4, v3
; GFX10-NEXT: v_fma_mix_f32 v5, v1, v4, neg(0) op_sel_hi:[1,0,0]
; GFX10-NEXT: v_mad_f32 v7, -v3, v5, v6
@@ -1983,12 +1991,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX11-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX11-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -2017,12 +2026,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX11-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX11-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
@@ -2051,12 +2061,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1150-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-TRUE16-NEXT: s_clause 0x1
-; GFX1150-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX1150-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX1150-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX1150-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1150-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1150-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -2086,12 +2097,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1150-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1150-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-FAKE16-NEXT: s_clause 0x1
-; GFX1150-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX1150-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX1150-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX1150-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1150-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX1150-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX1150-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1150-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
@@ -2121,12 +2133,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
-; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x1
; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
+; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
@@ -2156,12 +2169,13 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
; GFX1200-FAKE16-NEXT: global_load_u16 v2, v0, s[4:5] offset:8
-; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX1200-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v3, v3
+; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1200-FAKE16-NEXT: v_fma_mix_f32 v4, v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-FAKE16-NEXT: v_fma_mix_f32 v5, -v2, v4, v1 op_sel_hi:[1,0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 1a2a47330e16c..fdd37cf5dc953 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -2481,9 +2481,9 @@ define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0
; CIGFX89: ; %bb.0:
; CIGFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CIGFX89-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; CIGFX89-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4
; CIGFX89-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
; CIGFX89-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
-; CIGFX89-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4
; CIGFX89-NEXT: s_mov_b32 s7, 0xf000
; CIGFX89-NEXT: s_mov_b32 s6, -1
; CIGFX89-NEXT: s_waitcnt vmcnt(3)
@@ -2514,8 +2514,8 @@ define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:12
; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:12
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:8
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
@@ -2536,7 +2536,7 @@ define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: s_waitcnt vmcnt(2)
; GFX11-NEXT: buffer_store_b32 v34, off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2861,11 +2861,11 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX89-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
-; GFX89-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX89-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:4
; GFX89-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:8
; GFX89-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:12
+; GFX89-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:20
+; GFX89-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:16
; GFX89-NEXT: s_mov_b32 s7, 0xf000
; GFX89-NEXT: s_mov_b32 s6, -1
; GFX89-NEXT: s_waitcnt vmcnt(5)
@@ -2900,10 +2900,10 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x5
; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:20
; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:4
; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:12
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:20
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:16
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
@@ -2924,13 +2924,13 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: buffer_store_b32 v34, off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: buffer_store_b32 v35, off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: s_waitcnt vmcnt(2)
; GFX11-NEXT: buffer_store_b32 v36, off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2950,13 +2950,13 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; CI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
-; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
-; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; CI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:16
; CI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:12
; CI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:8
; CI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:4
+; CI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
+; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt vmcnt(7)
@@ -2987,13 +2987,13 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; VI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
-; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
-; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:12
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:4
+; VI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
+; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt vmcnt(7)
@@ -3024,13 +3024,13 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:24
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt vmcnt(7)
@@ -3063,13 +3063,13 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x8
; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:32
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:28
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:24
; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:16
; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:12
; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:32
+; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:28
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:24
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:20
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
@@ -3090,7 +3090,7 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: buffer_store_b128 v[36:39], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3193,13 +3193,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; CI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
+; CI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; CI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
+; CI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; CI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; CI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; CI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; CI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
-; CI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; CI-NEXT: s_waitcnt vmcnt(7)
; CI-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -3212,13 +3212,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; CI-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
-; CI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
-; CI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
-; CI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; CI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:64
; CI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:60
; CI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
; CI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52
+; CI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
+; CI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
+; CI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; CI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
; CI-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -3242,13 +3242,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
+; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; VI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; VI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; VI-NEXT: s_waitcnt vmcnt(7)
; VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -3261,13 +3261,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
-; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:64
; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
+; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
+; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -3291,13 +3291,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -3310,13 +3310,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; GFX9-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
-; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:64
; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
+; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
+; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
@@ -3339,26 +3339,26 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x10
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:48
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:44
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:40
-; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:64
-; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:60
; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:56
-; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:16
-; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:12
-; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:32
; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:28
; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:24
; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:20
+; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:16
+; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:12
+; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:64
+; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:60
+; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:56
; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:52
+; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:48
+; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:44
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:40
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:36
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_waitcnt vmcnt(11)
+; GFX11-NEXT: s_waitcnt vmcnt(16)
; GFX11-NEXT: buffer_store_b128 v[28:31], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[24:27], off, s[0:3], 0 dlc
@@ -3375,13 +3375,13 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: s_waitcnt vmcnt(12)
; GFX11-NEXT: buffer_store_b128 v[52:55], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(8)
; GFX11-NEXT: buffer_store_b128 v[48:51], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: buffer_store_b128 v[36:39], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3432,25 +3432,25 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:4
+; CI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
+; CI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
+; CI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; CI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112
; CI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108
; CI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104
; CI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100
-; CI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
-; CI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
-; CI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_dwordx4 v[32:35], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116
-; CI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
-; CI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
-; CI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; CI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:96
; CI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:92
; CI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:88
; CI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
+; CI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
+; CI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
+; CI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; CI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68
; CI-NEXT: buffer_store_dwordx4 v[36:39], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -3505,25 +3505,25 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
; VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:4
+; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
+; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
+; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100
-; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
-; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
-; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dwordx4 v[32:35], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:96
; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:92
; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:88
; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; VI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68
; VI-NEXT: buffer_store_dwordx4 v[36:39], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -3579,26 +3579,26 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
; GFX9-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:4
+; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
+; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
+; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100
-; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:128
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_store_dwordx4 v[32:35], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:96
; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:92
; GFX9-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:88
; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:84
+; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dwordx4 v[36:39], off, s[4:7], 0
@@ -3621,42 +3621,42 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1f
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:80
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:76
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:72
-; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:96
-; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:92
-; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:88
-; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:112
-; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:108
-; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:104
-; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:128
-; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:124
-; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:120
-; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:16
-; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:12
-; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:8
-; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:32
-; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:28
; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:24
-; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:48
-; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:44
-; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:40
; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:64
; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:60
; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:56
; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:52
+; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:48
+; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:44
+; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:40
; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:36
+; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:32
+; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:28
+; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:24
; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:20
+; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:16
+; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:12
+; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:128
+; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:124
+; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:120
; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:116
+; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:112
+; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:108
+; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:104
; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:100
+; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:96
+; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:92
+; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:88
; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:84
+; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:80
+; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:76
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:72
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:68
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: s_waitcnt vmcnt(15)
+; GFX11-NEXT: s_waitcnt vmcnt(32)
; GFX11-NEXT: buffer_store_b128 v[28:31], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[24:27], off, s[0:3], 0 dlc
@@ -3673,25 +3673,25 @@ define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1,
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(7)
+; GFX11-NEXT: s_waitcnt vmcnt(28)
; GFX11-NEXT: buffer_store_b128 v[84:87], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: s_waitcnt vmcnt(24)
; GFX11-NEXT: buffer_store_b128 v[80:83], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: s_waitcnt vmcnt(20)
; GFX11-NEXT: buffer_store_b128 v[68:71], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: s_waitcnt vmcnt(16)
; GFX11-NEXT: buffer_store_b128 v[64:67], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: s_waitcnt vmcnt(12)
; GFX11-NEXT: buffer_store_b128 v[52:55], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(8)
; GFX11-NEXT: buffer_store_b128 v[48:51], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: buffer_store_b128 v[36:39], off, s[0:3], 0 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3900,13 +3900,13 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
; CI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
-; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
-; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
; CI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64
-; CI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:52
+; CI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60
; CI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:56
-; CI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
+; CI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:52
+; CI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:48
; CI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40
+; CI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
; CI-NEXT: s_waitcnt vmcnt(7)
; CI-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -3973,13 +3973,13 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
; VI-NEXT: buffer_load_dword v31, off, s[0:3], s32
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: buffer_load_ubyte v32, off, s[0:3], s32 offset:48
-; VI-NEXT: buffer_load_ubyte v33, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_ubyte v34, off, s[0:3], s32 offset:64
-; VI-NEXT: buffer_load_ubyte v35, off, s[0:3], s32 offset:52
+; VI-NEXT: buffer_load_ubyte v33, off, s[0:3], s32 offset:60
; VI-NEXT: buffer_load_ubyte v36, off, s[0:3], s32 offset:56
-; VI-NEXT: buffer_load_ubyte v37, off, s[0:3], s32 offset:36
+; VI-NEXT: buffer_load_ubyte v35, off, s[0:3], s32 offset:52
+; VI-NEXT: buffer_load_ubyte v32, off, s[0:3], s32 offset:48
; VI-NEXT: buffer_load_ubyte v38, off, s[0:3], s32 offset:40
+; VI-NEXT: buffer_load_ubyte v37, off, s[0:3], s32 offset:36
; VI-NEXT: s_waitcnt vmcnt(7)
; VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -4046,13 +4046,13 @@ define void @void_func_v32i32_v16i8(<32 x i32> %arg0, <16 x i8> %arg1) #0 {
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
-; GFX9-NEXT: buffer_load_ubyte v32, off, s[0:3], s32 offset:48
-; GFX9-NEXT: buffer_load_ubyte v33, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ubyte v34, off, s[0:3], s32 offset:64
-; GFX9-NEXT: buffer_load_ubyte v35, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ubyte v33, off, s[0:3], s32 offset:60
; GFX9-NEXT: buffer_load_ubyte v36, off, s[0:3], s32 offset:56
-; GFX9-NEXT: buffer_load_ubyte v37, off, s[0:3], s32 offset:36
+; GFX9-NEXT: buffer_load_ubyte v35, off, s[0:3], s32 offset:52
+; GFX9-NEXT: buffer_load_ubyte v32, off, s[0:3], s32 offset:48
; GFX9-NEXT: buffer_load_ubyte v38, off, s[0:3], s32 offset:40
+; GFX9-NEXT: buffer_load_ubyte v37, off, s[0:3], s32 offset:36
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: buffer_store_dwordx4 v[28:31], off, s[4:7], 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index f805b11d35db5..b8dacc923e27b 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -2419,27 +2419,27 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
-; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
-; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:72
-; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76
-; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80
-; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:84
-; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:88
; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:92
+; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:88
+; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:84
+; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:80
+; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:76
+; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:72
+; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68
+; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64
; GFX10-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:120
; GFX10-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:116
; GFX10-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:112
; GFX10-NEXT: buffer_store_dword v28, v0, s[0:3], 0 offen offset:108
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128
-; GFX10-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:132
-; GFX10-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:136
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:140
-; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:144
-; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:148
-; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:152
; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:156
+; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:152
+; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:148
+; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:144
+; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:140
+; GFX10-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:136
+; GFX10-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:132
+; GFX10-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:128
; GFX10-NEXT: buffer_store_dword v27, v0, s[0:3], 0 offen offset:104
; GFX10-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:100
; GFX10-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:96
@@ -2450,14 +2450,14 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
; GFX10-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:76
; GFX10-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:72
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96
-; GFX10-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100
-; GFX10-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:104
-; GFX10-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:108
-; GFX10-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
-; GFX10-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116
-; GFX10-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120
; GFX10-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:124
+; GFX10-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:120
+; GFX10-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116
+; GFX10-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:112
+; GFX10-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:108
+; GFX10-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:104
+; GFX10-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100
+; GFX10-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:96
; GFX10-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:68
; GFX10-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64
; GFX10-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60
@@ -2467,14 +2467,14 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
; GFX10-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:44
; GFX10-NEXT: buffer_store_dword v11, v0, s[0:3], 0 offen offset:40
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:32
-; GFX10-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
-; GFX10-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
-; GFX10-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
-; GFX10-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
-; GFX10-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52
-; GFX10-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
; GFX10-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:60
+; GFX10-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56
+; GFX10-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52
+; GFX10-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48
+; GFX10-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44
+; GFX10-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:40
+; GFX10-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:36
+; GFX10-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:32
; GFX10-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:36
; GFX10-NEXT: buffer_store_dword v9, v0, s[0:3], 0 offen offset:32
; GFX10-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:28
@@ -2569,24 +2569,24 @@ define amdgpu_gfx <72 x i32> @return_72xi32(<72 x i32> %val) #1 {
; GFX11-NEXT: ; meta instruction
; GFX11-NEXT: scratch_store_b32 off, v63, s32 offset:164
; GFX11-NEXT: s_clause 0x11
-; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:16
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:12
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:8
-; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:32
-; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:28
-; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:24
-; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:48
-; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:44
-; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:40
-; GFX11-NEXT: scratch_load_b32 v40, off, s32 offset:64
-; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:60
-; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:56
-; GFX11-NEXT: scratch_load_b32 v44, off, s32 offset:80
-; GFX11-NEXT: scratch_load_b32 v43, off, s32 offset:76
-; GFX11-NEXT: scratch_load_b32 v42, off, s32 offset:72
; GFX11-NEXT: scratch_load_b32 v59, off, s32 offset:96
; GFX11-NEXT: scratch_load_b32 v58, off, s32 offset:92
; GFX11-NEXT: scratch_load_b32 v57, off, s32 offset:88
+; GFX11-NEXT: scratch_load_b32 v44, off, s32 offset:80
+; GFX11-NEXT: scratch_load_b32 v43, off, s32 offset:76
+; GFX11-NEXT: scratch_load_b32 v42, off, s32 offset:72
+; GFX11-NEXT: scratch_load_b32 v40, off, s32 offset:64
+; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:60
+; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:56
+; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:48
+; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:44
+; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:40
+; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:32
+; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:28
+; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:24
+; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:16
+; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:12
+; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:8
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: scratch_store_b128 v0, v[21:24], off offset:80
; GFX11-NEXT: scratch_store_b128 v0, v[17:20], off offset:64
@@ -2771,6 +2771,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX9-NEXT: v_mov_b32_e32 v30, 0
; GFX9-NEXT: v_mov_b32_e32 v31, 0
; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]
+; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:516
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:636
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640
; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644
@@ -2811,8 +2812,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:516
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(40)
; GFX9-NEXT: buffer_store_dword v9, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:520
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -3045,6 +3045,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX10-NEXT: s_mov_b32 s36, return_72xi32 at abs32@lo
; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37]
; GFX10-NEXT: s_clause 0x28
+; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:516
; GFX10-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:636
; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:640
; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:644
@@ -3085,8 +3086,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX10-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:788
; GFX10-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:792
; GFX10-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:796
-; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:516
-; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: s_waitcnt vmcnt(40)
; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:1536 ; 4-byte Folded Spill
; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:520
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -3308,6 +3308,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: s_clause 0xb
+; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:512
; GFX11-NEXT: scratch_load_b128 v[43:46], off, s33 offset:624
; GFX11-NEXT: scratch_load_b128 v[47:50], off, s33 offset:640
; GFX11-NEXT: scratch_load_b128 v[16:19], off, s33 offset:784
@@ -3319,22 +3320,19 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: scratch_load_b128 v[36:39], off, s33 offset:704
; GFX11-NEXT: scratch_load_b128 v[32:35], off, s33 offset:688
; GFX11-NEXT: scratch_load_b128 v[28:31], off, s33 offset:672
-; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:512
-; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: s_waitcnt vmcnt(5)
; GFX11-NEXT: v_mov_b32_e32 v6, 24
; GFX11-NEXT: s_add_i32 s2, s32, 0xa0
; GFX11-NEXT: s_add_i32 s3, s32, 0x90
; GFX11-NEXT: s_add_i32 s35, s32, 0x80
; GFX11-NEXT: s_add_i32 s36, s32, 0x70
; GFX11-NEXT: s_add_i32 s37, s32, 0x6c
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:528
; GFX11-NEXT: s_waitcnt vmcnt(4)
; GFX11-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v53, v26
; GFX11-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v42, v2
-; GFX11-NEXT: v_mov_b32_e32 v51, v24
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill
-; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:528
-; GFX11-NEXT: v_mov_b32_e32 v52, v25
+; GFX11-NEXT: v_dual_mov_b32 v51, v24 :: v_dual_mov_b32 v52, v25
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:544
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
index 6b3754cc039a5..0900af83df80a 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
@@ -67,24 +67,24 @@ body: |
; GFX11-NEXT: renamable $vgpr5 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr5, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: renamable $vgpr6 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr6, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: }
- ; GFX11-NEXT: BUNDLE implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr8, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr7, implicit killed $vgpr9, implicit $sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec, implicit killed $vgpr10, implicit killed $vgpr11, implicit killed $vgpr8, implicit killed $vgpr12, implicit killed $vgpr13, implicit killed $vgpr14, implicit killed $vgpr7 :: (dereferenceable invariant load (s16), align 1, addrspace 8) {
+ ; GFX11-NEXT: BUNDLE implicit-def $vgpr9, implicit-def $vgpr12, implicit-def $vgpr10, implicit-def $vgpr13, implicit-def $vgpr11, implicit-def $vgpr14, implicit-def $vgpr8, implicit-def $vgpr7, implicit killed $vgpr9, implicit $sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec, implicit killed $vgpr12, implicit killed $vgpr10, implicit killed $vgpr13, implicit killed $vgpr11, implicit killed $vgpr14, implicit killed $vgpr8, implicit killed $vgpr7 :: (dereferenceable invariant load (s16), align 1, addrspace 8) {
; GFX11-NEXT: S_CLAUSE 7
; GFX11-NEXT: renamable $vgpr9 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr9, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
- ; GFX11-NEXT: renamable $vgpr10 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr10, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
- ; GFX11-NEXT: renamable $vgpr11 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr11, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
- ; GFX11-NEXT: renamable $vgpr8 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr8, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: renamable $vgpr12 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr12, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
+ ; GFX11-NEXT: renamable $vgpr10 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr10, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: renamable $vgpr13 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr13, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
+ ; GFX11-NEXT: renamable $vgpr11 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr11, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: renamable $vgpr14 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr14, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
+ ; GFX11-NEXT: renamable $vgpr8 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr8, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: renamable $vgpr7 = BUFFER_LOAD_USHORT_IDXEN killed renamable $vgpr7, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s16), align 1, addrspace 8)
; GFX11-NEXT: }
; GFX11-NEXT: S_WAITCNT .Vmcnt_8
; GFX11-NEXT: renamable $vgpr5 = V_PERM_B32_e64 killed $vgpr6, killed $vgpr5, 84148480, implicit $exec
- ; GFX11-NEXT: S_WAITCNT .Vmcnt_3
+ ; GFX11-NEXT: S_WAITCNT .Vmcnt_6
; GFX11-NEXT: renamable $vgpr6 = V_PERM_B32_e64 killed $vgpr12, killed $vgpr9, 84148480, implicit $exec
- ; GFX11-NEXT: S_WAITCNT .Vmcnt_2
+ ; GFX11-NEXT: S_WAITCNT .Vmcnt_4
; GFX11-NEXT: renamable $vgpr9 = V_PERM_B32_e64 killed $vgpr13, killed $vgpr10, 84148480, implicit $exec
- ; GFX11-NEXT: S_WAITCNT .Vmcnt_1
+ ; GFX11-NEXT: S_WAITCNT .Vmcnt_2
; GFX11-NEXT: renamable $vgpr10 = V_PERM_B32_e64 killed $vgpr14, killed $vgpr11, 84148480, implicit $exec
; GFX11-NEXT: S_WAITCNT .Vmcnt_0
; GFX11-NEXT: renamable $vgpr7 = V_PERM_B32_e64 killed $vgpr7, killed $vgpr8, 84148480, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index 5e6c34c085aa4..d066c05819685 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -643,15 +643,15 @@ define amdgpu_kernel void @udot2_8(ptr addrspace(1) %src1,
; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]
; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0100
; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s0
-; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]
@@ -665,14 +665,14 @@ define amdgpu_kernel void @udot2_8(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_clause 0x1
-; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]
+; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]
; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]
; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
-; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100
-; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0100
+; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
+; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 127f4996b7676..f17261909d99d 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -1504,6 +1504,7 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
+; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -1520,12 +1521,12 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: buffer_load_dword v13, off, s[20:23], 0 offset:56 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v14, off, s[20:23], 0 offset:60 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_add_i32 m0, s2, 0xfffffe00
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movrels_b32_e32 v0, v0
; NOOPT-NEXT: buffer_store_dword v0, off, s[20:23], 0 offset:72 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
@@ -3895,6 +3896,7 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v31, 9
; NOOPT-NEXT: v_readlane_b32 s1, v31, 10
+; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -3919,12 +3921,12 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_add_i32 m0, s2, 0xfffffe00
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[20:23], 0 offset:140 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[20:23], 0 offset:144 ; 4-byte Folded Spill
@@ -3987,41 +3989,52 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: v_readlane_b32 s4, v31, 0
; NOOPT-NEXT: v_readlane_b32 s5, v31, 1
; NOOPT-NEXT: v_readlane_b32 s6, v31, 2
-; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v18, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v22, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v26, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
+; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_mov_b32_e32 v4, v18
; NOOPT-NEXT: v_mov_b32_e32 v5, v17
+; NOOPT-NEXT: s_waitcnt vmcnt(13)
; NOOPT-NEXT: v_mov_b32_e32 v6, v16
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
+; NOOPT-NEXT: s_waitcnt vmcnt(11)
; NOOPT-NEXT: v_mov_b32_e32 v1, v22
+; NOOPT-NEXT: s_waitcnt vmcnt(10)
; NOOPT-NEXT: v_mov_b32_e32 v2, v21
+; NOOPT-NEXT: s_waitcnt vmcnt(9)
; NOOPT-NEXT: v_mov_b32_e32 v3, v20
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
+; NOOPT-NEXT: s_waitcnt vmcnt(7)
; NOOPT-NEXT: v_mov_b32_e32 v12, v26
+; NOOPT-NEXT: s_waitcnt vmcnt(6)
; NOOPT-NEXT: v_mov_b32_e32 v13, v25
+; NOOPT-NEXT: s_waitcnt vmcnt(5)
; NOOPT-NEXT: v_mov_b32_e32 v14, v24
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v8, v23
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(3)
; NOOPT-NEXT: v_mov_b32_e32 v9, v30
+; NOOPT-NEXT: s_waitcnt vmcnt(2)
; NOOPT-NEXT: v_mov_b32_e32 v10, v29
+; NOOPT-NEXT: s_waitcnt vmcnt(1)
; NOOPT-NEXT: v_mov_b32_e32 v11, v28
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v15, v27
; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v16, v11
@@ -4384,6 +4397,7 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v31, 9
; NOOPT-NEXT: v_readlane_b32 s1, v31, 10
+; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -4408,12 +4422,12 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_add_i32 m0, s2, -16
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[20:23], 0 offset:140 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[20:23], 0 offset:144 ; 4-byte Folded Spill
@@ -4476,41 +4490,52 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: v_readlane_b32 s4, v31, 0
; NOOPT-NEXT: v_readlane_b32 s5, v31, 1
; NOOPT-NEXT: v_readlane_b32 s6, v31, 2
-; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v18, off, s[20:23], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v22, off, s[20:23], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v21, off, s[20:23], 0 offset:164 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v20, off, s[20:23], 0 offset:160 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v19, off, s[20:23], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v26, off, s[20:23], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v25, off, s[20:23], 0 offset:180 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v24, off, s[20:23], 0 offset:176 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v23, off, s[20:23], 0 offset:172 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
+; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v27, off, s[20:23], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_mov_b32_e32 v4, v18
; NOOPT-NEXT: v_mov_b32_e32 v5, v17
+; NOOPT-NEXT: s_waitcnt vmcnt(13)
; NOOPT-NEXT: v_mov_b32_e32 v6, v16
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v0, v15
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
+; NOOPT-NEXT: s_waitcnt vmcnt(11)
; NOOPT-NEXT: v_mov_b32_e32 v1, v22
+; NOOPT-NEXT: s_waitcnt vmcnt(10)
; NOOPT-NEXT: v_mov_b32_e32 v2, v21
+; NOOPT-NEXT: s_waitcnt vmcnt(9)
; NOOPT-NEXT: v_mov_b32_e32 v3, v20
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
+; NOOPT-NEXT: s_waitcnt vmcnt(7)
; NOOPT-NEXT: v_mov_b32_e32 v12, v26
+; NOOPT-NEXT: s_waitcnt vmcnt(6)
; NOOPT-NEXT: v_mov_b32_e32 v13, v25
+; NOOPT-NEXT: s_waitcnt vmcnt(5)
; NOOPT-NEXT: v_mov_b32_e32 v14, v24
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v8, v23
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(3)
; NOOPT-NEXT: v_mov_b32_e32 v9, v30
+; NOOPT-NEXT: s_waitcnt vmcnt(2)
; NOOPT-NEXT: v_mov_b32_e32 v10, v29
+; NOOPT-NEXT: s_waitcnt vmcnt(1)
; NOOPT-NEXT: v_mov_b32_e32 v11, v28
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v15, v27
; NOOPT-NEXT: ; kill: def $vgpr15 killed $vgpr15 def $vgpr15_vgpr16_vgpr17_vgpr18 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v16, v11
@@ -4939,6 +4964,7 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v17, 23
; NOOPT-NEXT: v_readlane_b32 s1, v17, 24
+; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:72 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:12 ; 4-byte Folded Reload
@@ -4955,12 +4981,12 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: buffer_load_dword v13, off, s[36:39], 0 offset:56 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v14, off, s[36:39], 0 offset:60 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v15, off, s[36:39], 0 offset:64 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:72 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movrels_b32_e32 v0, v0
; NOOPT-NEXT: buffer_store_dword v0, off, s[36:39], 0 offset:80 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
@@ -5054,6 +5080,7 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v17, 28
; NOOPT-NEXT: v_readlane_b32 s1, v17, 29
+; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:68 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:84 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:88 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:92 ; 4-byte Folded Reload
@@ -5070,12 +5097,12 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: buffer_load_dword v13, off, s[36:39], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v14, off, s[36:39], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v15, off, s[36:39], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movrels_b32_e32 v0, v0
; NOOPT-NEXT: buffer_store_dword v0, off, s[36:39], 0 offset:148 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[2:3], s[0:1]
@@ -5105,10 +5132,10 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: v_readlane_b32 s5, v17, 1
; NOOPT-NEXT: v_readlane_b32 s6, v17, 2
; NOOPT-NEXT: v_readlane_b32 s7, v17, 3
-; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:76 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:148 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:148 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:76 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(2)
; NOOPT-NEXT: buffer_store_dword v2, off, s[4:7], 0
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v1, off, s[4:7], 0
@@ -5675,6 +5702,7 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v32, 11
; NOOPT-NEXT: v_readlane_b32 s1, v32, 12
+; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:80 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:12 ; 4-byte Folded Reload
@@ -5699,12 +5727,12 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:72 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[28:31], 0 offset:88 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[28:31], 0 offset:92 ; 4-byte Folded Spill
@@ -5813,6 +5841,7 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v32, 15
; NOOPT-NEXT: v_readlane_b32 s1, v32, 16
+; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:76 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:152 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:160 ; 4-byte Folded Reload
@@ -5837,12 +5866,12 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:212 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:216 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:76 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[28:31], 0 offset:220 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[28:31], 0 offset:224 ; 4-byte Folded Spill
@@ -5907,42 +5936,52 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: v_readlane_b32 s3, v32, 8
; NOOPT-NEXT: v_readlane_b32 s8, v32, 0
; NOOPT-NEXT: v_readlane_b32 s9, v32, 1
-; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:84 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:220 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:224 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[28:31], 0 offset:228 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v19, off, s[28:31], 0 offset:232 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v20, off, s[28:31], 0 offset:236 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[28:31], 0 offset:240 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[28:31], 0 offset:244 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v18, off, s[28:31], 0 offset:228 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:224 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:220 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v23, off, s[28:31], 0 offset:248 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v24, off, s[28:31], 0 offset:252 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[28:31], 0 offset:256 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[28:31], 0 offset:260 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v22, off, s[28:31], 0 offset:244 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v21, off, s[28:31], 0 offset:240 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v20, off, s[28:31], 0 offset:236 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v27, off, s[28:31], 0 offset:264 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v28, off, s[28:31], 0 offset:268 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[28:31], 0 offset:272 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[28:31], 0 offset:276 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v26, off, s[28:31], 0 offset:260 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v25, off, s[28:31], 0 offset:256 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v24, off, s[28:31], 0 offset:252 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 offset:280 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
+; NOOPT-NEXT: buffer_load_dword v30, off, s[28:31], 0 offset:276 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v29, off, s[28:31], 0 offset:272 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v28, off, s[28:31], 0 offset:268 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:84 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_mov_b32_e32 v5, v19
; NOOPT-NEXT: v_mov_b32_e32 v6, v18
; NOOPT-NEXT: v_mov_b32_e32 v7, v17
+; NOOPT-NEXT: s_waitcnt vmcnt(13)
; NOOPT-NEXT: v_mov_b32_e32 v1, v16
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v2, v23
+; NOOPT-NEXT: s_waitcnt vmcnt(11)
; NOOPT-NEXT: v_mov_b32_e32 v3, v22
+; NOOPT-NEXT: s_waitcnt vmcnt(10)
; NOOPT-NEXT: v_mov_b32_e32 v4, v21
+; NOOPT-NEXT: s_waitcnt vmcnt(9)
; NOOPT-NEXT: v_mov_b32_e32 v8, v20
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v13, v27
+; NOOPT-NEXT: s_waitcnt vmcnt(7)
; NOOPT-NEXT: v_mov_b32_e32 v14, v26
+; NOOPT-NEXT: s_waitcnt vmcnt(6)
; NOOPT-NEXT: v_mov_b32_e32 v15, v25
+; NOOPT-NEXT: s_waitcnt vmcnt(5)
; NOOPT-NEXT: v_mov_b32_e32 v9, v24
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v10, v31
+; NOOPT-NEXT: s_waitcnt vmcnt(3)
; NOOPT-NEXT: v_mov_b32_e32 v11, v30
+; NOOPT-NEXT: s_waitcnt vmcnt(2)
; NOOPT-NEXT: v_mov_b32_e32 v12, v29
+; NOOPT-NEXT: s_waitcnt vmcnt(1)
; NOOPT-NEXT: v_mov_b32_e32 v16, v28
; NOOPT-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17_vgpr18_vgpr19 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v17, v12
@@ -8881,6 +8920,7 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v18, 6
; NOOPT-NEXT: v_readlane_b32 s1, v18, 7
+; NOOPT-NEXT: buffer_load_dword v17, off, s[24:27], 0 offset:80 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:12 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[24:27], 0 offset:16 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[24:27], 0 offset:20 ; 4-byte Folded Reload
@@ -8905,12 +8945,12 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[24:27], 0 offset:72 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[24:27], 0 offset:76 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[24:27], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v0, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[24:27], 0 offset:84 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[24:27], 0 offset:88 ; 4-byte Folded Spill
@@ -9370,6 +9410,7 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v33, 11
; NOOPT-NEXT: v_readlane_b32 s1, v33, 12
+; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:132 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
@@ -9394,12 +9435,12 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v15, off, s[16:19], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[16:19], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:132 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; NOOPT-NEXT: s_mov_b32 m0, s2
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_movreld_b32_e32 v1, v16
; NOOPT-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:148 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[16:19], 0 offset:152 ; 4-byte Folded Spill
@@ -9461,43 +9502,52 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: v_readlane_b32 s3, v33, 8
; NOOPT-NEXT: v_readlane_b32 s8, v33, 3
; NOOPT-NEXT: v_readlane_b32 s9, v33, 4
-; NOOPT-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v5, off, s[16:19], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v18, off, s[16:19], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v19, off, s[16:19], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v20, off, s[16:19], 0 offset:160 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v21, off, s[16:19], 0 offset:164 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v22, off, s[16:19], 0 offset:168 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v23, off, s[16:19], 0 offset:172 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v19, off, s[16:19], 0 offset:156 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v18, off, s[16:19], 0 offset:152 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v24, off, s[16:19], 0 offset:176 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v25, off, s[16:19], 0 offset:180 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v26, off, s[16:19], 0 offset:184 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v27, off, s[16:19], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v23, off, s[16:19], 0 offset:172 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v22, off, s[16:19], 0 offset:168 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v21, off, s[16:19], 0 offset:164 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v28, off, s[16:19], 0 offset:192 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v29, off, s[16:19], 0 offset:196 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v30, off, s[16:19], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v31, off, s[16:19], 0 offset:204 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v27, off, s[16:19], 0 offset:188 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v26, off, s[16:19], 0 offset:184 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v25, off, s[16:19], 0 offset:180 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 offset:208 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt vmcnt(12)
+; NOOPT-NEXT: buffer_load_dword v31, off, s[16:19], 0 offset:204 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v30, off, s[16:19], 0 offset:200 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v29, off, s[16:19], 0 offset:196 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:136 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v5, off, s[16:19], 0 offset:140 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_mov_b32_e32 v6, v20
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
; NOOPT-NEXT: v_mov_b32_e32 v8, v18
; NOOPT-NEXT: v_mov_b32_e32 v0, v17
-; NOOPT-NEXT: s_waitcnt vmcnt(8)
+; NOOPT-NEXT: s_waitcnt vmcnt(13)
; NOOPT-NEXT: v_mov_b32_e32 v1, v24
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v2, v23
+; NOOPT-NEXT: s_waitcnt vmcnt(11)
; NOOPT-NEXT: v_mov_b32_e32 v3, v22
+; NOOPT-NEXT: s_waitcnt vmcnt(10)
; NOOPT-NEXT: v_mov_b32_e32 v9, v21
-; NOOPT-NEXT: s_waitcnt vmcnt(4)
+; NOOPT-NEXT: s_waitcnt vmcnt(9)
; NOOPT-NEXT: v_mov_b32_e32 v14, v28
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v15, v27
+; NOOPT-NEXT: s_waitcnt vmcnt(7)
; NOOPT-NEXT: v_mov_b32_e32 v16, v26
+; NOOPT-NEXT: s_waitcnt vmcnt(6)
; NOOPT-NEXT: v_mov_b32_e32 v10, v25
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: s_waitcnt vmcnt(5)
; NOOPT-NEXT: v_mov_b32_e32 v11, v32
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v12, v31
+; NOOPT-NEXT: s_waitcnt vmcnt(3)
; NOOPT-NEXT: v_mov_b32_e32 v13, v30
+; NOOPT-NEXT: s_waitcnt vmcnt(2)
; NOOPT-NEXT: v_mov_b32_e32 v17, v29
; NOOPT-NEXT: ; kill: def $vgpr17 killed $vgpr17 def $vgpr17_vgpr18_vgpr19_vgpr20 killed $exec
; NOOPT-NEXT: v_mov_b32_e32 v18, v13
@@ -9506,6 +9556,7 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: s_mov_b64 s[4:5], 32
; NOOPT-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; NOOPT-NEXT: s_mov_b64 s[6:7], s[8:9]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v12, v5
; NOOPT-NEXT: v_mov_b32_e32 v11, v4
; NOOPT-NEXT: buffer_store_dwordx4 v[17:20], v[11:12], s[4:7], 0 addr64 offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 4f0083bdbd915..99d1b0d8b8b79 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -1015,16 +1015,18 @@ define amdgpu_kernel void @half8_inselt(ptr addrspace(1) %out, <8 x half> %vec,
; GCN-O0-NEXT: v_mov_b32_e32 v0, s3
; GCN-O0-NEXT: buffer_load_dword v0, v0, s[12:15], 0 offen offset:4
; GCN-O0-NEXT: v_mov_b32_e32 v1, s2
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 offset:4
; GCN-O0-NEXT: buffer_load_dword v1, v1, s[12:15], 0 offen offset:4
; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 offset:4
; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(2)
; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GCN-O0-NEXT: s_endpgm
entry:
@@ -1875,11 +1877,11 @@ define amdgpu_kernel void @byte16_inselt(ptr addrspace(1) %out, <16 x i8> %vec,
; GCN-O0-NEXT: buffer_load_ubyte v16, v0, s[24:27], 0 offen offset:2
; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
; GCN-O0-NEXT: buffer_load_ubyte v17, v0, s[24:27], 0 offen offset:1
+; GCN-O0-NEXT: buffer_load_ubyte v10, off, s[24:27], 0 offset:8
; GCN-O0-NEXT: buffer_load_ubyte v2, off, s[24:27], 0
; GCN-O0-NEXT: buffer_load_ubyte v3, off, s[24:27], 0 offset:1
; GCN-O0-NEXT: buffer_load_ubyte v4, off, s[24:27], 0 offset:2
; GCN-O0-NEXT: buffer_load_ubyte v6, off, s[24:27], 0 offset:4
-; GCN-O0-NEXT: buffer_load_ubyte v10, off, s[24:27], 0 offset:8
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
; GCN-O0-NEXT: s_mov_b32 s5, s12
@@ -1968,7 +1970,7 @@ define amdgpu_kernel void @byte16_inselt(ptr addrspace(1) %out, <16 x i8> %vec,
; GCN-O0-NEXT: flat_store_byte v[0:1], v11
; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
-; GCN-O0-NEXT: s_waitcnt vmcnt(7)
+; GCN-O0-NEXT: s_waitcnt vmcnt(11)
; GCN-O0-NEXT: flat_store_byte v[0:1], v10
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -2013,6 +2015,7 @@ define amdgpu_kernel void @byte16_inselt(ptr addrspace(1) %out, <16 x i8> %vec,
; GCN-O0-NEXT: flat_store_byte v[0:1], v7
; GCN-O0-NEXT: v_mov_b32_e32 v0, s2
; GCN-O0-NEXT: v_mov_b32_e32 v1, s3
+; GCN-O0-NEXT: s_waitcnt vmcnt(11)
; GCN-O0-NEXT: flat_store_byte v[0:1], v6
; GCN-O0-NEXT: s_mov_b32 s2, s0
; GCN-O0-NEXT: s_mov_b32 s3, s1
@@ -3626,6 +3629,7 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:212 ; 4-byte Folded Spill
; GCN-O0-NEXT: v_mov_b32_e32 v32, v1
; GCN-O0-NEXT: buffer_store_dword v32, off, s[56:59], 0 offset:156 ; 4-byte Folded Spill
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:156 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v29, off, s[56:59], 0 offset:212 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v28, off, s[56:59], 0 offset:208 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v27, off, s[56:59], 0 offset:204 ; 4-byte Folded Reload
@@ -3640,27 +3644,40 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: buffer_load_dword v18, off, s[56:59], 0 offset:168 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v17, off, s[56:59], 0 offset:164 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v16, off, s[56:59], 0 offset:160 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:156 ; 4-byte Folded Reload
; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_mov_b32_e32 v1, v2
; GCN-O0-NEXT: v_mov_b32_e32 v30, v1
; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 killed $exec
; GCN-O0-NEXT: v_mov_b32_e32 v1, v30
+; GCN-O0-NEXT: s_waitcnt vmcnt(13)
; GCN-O0-NEXT: v_mov_b32_e32 v2, v29
+; GCN-O0-NEXT: s_waitcnt vmcnt(12)
; GCN-O0-NEXT: v_mov_b32_e32 v3, v28
+; GCN-O0-NEXT: s_waitcnt vmcnt(11)
; GCN-O0-NEXT: v_mov_b32_e32 v4, v27
+; GCN-O0-NEXT: s_waitcnt vmcnt(10)
; GCN-O0-NEXT: v_mov_b32_e32 v5, v26
+; GCN-O0-NEXT: s_waitcnt vmcnt(9)
; GCN-O0-NEXT: v_mov_b32_e32 v6, v25
+; GCN-O0-NEXT: s_waitcnt vmcnt(8)
; GCN-O0-NEXT: v_mov_b32_e32 v7, v24
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_mov_b32_e32 v8, v23
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_mov_b32_e32 v9, v22
+; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_mov_b32_e32 v10, v21
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: v_mov_b32_e32 v11, v20
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: v_mov_b32_e32 v12, v19
+; GCN-O0-NEXT: s_waitcnt vmcnt(2)
; GCN-O0-NEXT: v_mov_b32_e32 v13, v18
+; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_mov_b32_e32 v14, v17
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_mov_b32_e32 v15, v16
; GCN-O0-NEXT: v_mov_b32_e32 v16, v15
; GCN-O0-NEXT: v_mov_b32_e32 v17, v14
@@ -3724,6 +3741,10 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v41, v2
; GCN-O0-NEXT: v_mov_b32_e32 v46, v1
; GCN-O0-NEXT: v_mov_b32_e32 v42, v0
+; GCN-O0-NEXT: buffer_load_dword v5, off, s[56:59], 0 offset:20 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v4, off, s[56:59], 0 offset:16 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[56:59], 0 offset:12 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v36, off, s[56:59], 0 offset:144 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v7, off, s[56:59], 0 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v8, off, s[56:59], 0 offset:32 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v9, off, s[56:59], 0 offset:36 ; 4-byte Folded Reload
@@ -3753,16 +3774,12 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: buffer_load_dword v33, off, s[56:59], 0 offset:132 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v34, off, s[56:59], 0 offset:136 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v35, off, s[56:59], 0 offset:140 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v36, off, s[56:59], 0 offset:144 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v37, off, s[56:59], 0 offset:148 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v38, off, s[56:59], 0 offset:152 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[56:59], 0 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v5, off, s[56:59], 0 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v4, off, s[56:59], 0 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[56:59], 0 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[56:59], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[56:59], 0 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[56:59], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: ; kill: def $vgpr42 killed $vgpr42 def $vgpr42_vgpr43_vgpr44_vgpr45 killed $exec
; GCN-O0-NEXT: v_mov_b32_e32 v43, v46
; GCN-O0-NEXT: v_mov_b32_e32 v44, v41
@@ -3775,11 +3792,9 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v41, v49
; GCN-O0-NEXT: v_mov_b32_e32 v42, v48
; GCN-O0-NEXT: v_mov_b32_e32 v43, v47
-; GCN-O0-NEXT: s_waitcnt vmcnt(6)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_mov_b32_e32 v44, v5
-; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_mov_b32_e32 v45, v4
-; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: v_mov_b32_e32 v46, v3
; GCN-O0-NEXT: v_mov_b32_e32 v3, v46
; GCN-O0-NEXT: v_mov_b32_e32 v4, v45
@@ -3820,6 +3835,7 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
; GCN-O0-NEXT: flat_store_dwordx4 v[3:4], v[39:42]
; GCN-O0-NEXT: v_mov_b32_e32 v3, v36
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_mov_b32_e32 v7, v35
; GCN-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GCN-O0-NEXT: v_mov_b32_e32 v8, v3
@@ -3843,13 +3859,15 @@ define amdgpu_kernel void @double15_inselt(ptr addrspace(1) %out, <15 x double>
; GCN-O0-NEXT: v_mov_b32_e32 v3, s2
; GCN-O0-NEXT: flat_store_dwordx2 v[3:4], v[7:8]
; GCN-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3_vgpr4_vgpr5 killed $exec
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_mov_b32_e32 v3, v6
-; GCN-O0-NEXT: s_waitcnt vmcnt(5)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_mov_b32_e32 v4, v1
-; GCN-O0-NEXT: s_waitcnt vmcnt(4)
+; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_mov_b32_e32 v5, v0
; GCN-O0-NEXT: v_mov_b32_e32 v0, s0
; GCN-O0-NEXT: v_mov_b32_e32 v1, s1
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GCN-O0-NEXT: s_endpgm
entry:
@@ -7197,20 +7215,23 @@ define amdgpu_kernel void @bit128_inselt(ptr addrspace(1) %out, <128 x i1> %vec,
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, 1
; GCN-O0-NEXT: v_writelane_b32 v9, s0, 51
; GCN-O0-NEXT: v_writelane_b32 v9, s1, 52
+; GCN-O0-NEXT: buffer_load_ubyte v7, off, s[96:99], 0 offset:4
; GCN-O0-NEXT: buffer_load_ubyte v6, off, s[96:99], 0
; GCN-O0-NEXT: buffer_load_ubyte v5, off, s[96:99], 0 offset:1
; GCN-O0-NEXT: buffer_load_ubyte v4, off, s[96:99], 0 offset:2
-; GCN-O0-NEXT: buffer_load_ubyte v7, off, s[96:99], 0 offset:4
; GCN-O0-NEXT: buffer_load_ubyte v3, off, s[96:99], 0 offset:8
; GCN-O0-NEXT: buffer_load_ubyte v2, off, s[96:99], 0 offset:16
; GCN-O0-NEXT: buffer_load_ubyte v1, off, s[96:99], 0 offset:32
-; GCN-O0-NEXT: s_waitcnt vmcnt(3)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_and_b32_e64 v7, 1, v7
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[12:13], v7, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(5)
; GCN-O0-NEXT: v_and_b32_e64 v6, 1, v6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[18:19], v6, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: v_and_b32_e64 v5, 1, v5
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[20:21], v5, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: v_and_b32_e64 v4, 1, v4
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[22:23], v4, 1
; GCN-O0-NEXT: s_waitcnt vmcnt(2)
@@ -8225,6 +8246,7 @@ define amdgpu_ps <32 x float> @float32_inselt_vec(<32 x float> %vec, i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v63, 2
; GCN-O0-NEXT: v_readlane_b32 s1, v63, 3
+; GCN-O0-NEXT: buffer_load_dword v33, off, s[8:11], 0 offset:264 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[8:11], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:12 ; 4-byte Folded Reload
@@ -8258,12 +8280,12 @@ define amdgpu_ps <32 x float> @float32_inselt_vec(<32 x float> %vec, i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v30, off, s[8:11], 0 offset:124 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v31, off, s[8:11], 0 offset:128 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v32, off, s[8:11], 0 offset:132 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v33, off, s[8:11], 0 offset:264 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s2, v33
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v33
; GCN-O0-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GCN-O0-NEXT: s_mov_b32 m0, s2
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v32
; GCN-O0-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:268 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:272 ; 4-byte Folded Spill
@@ -8554,6 +8576,7 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v32, 4
; GCN-O0-NEXT: v_readlane_b32 s5, v32, 5
+; GCN-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -8571,12 +8594,12 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v17
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v17
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v16
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
@@ -8633,6 +8656,7 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v32, 0
; GCN-O0-NEXT: v_readlane_b32 s5, v32, 1
+; GCN-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
@@ -8649,9 +8673,8 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s6, 1
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_add_u32_e64 v16, s[6:7], v16, s6
; GCN-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b32 s4, s5
@@ -8665,16 +8688,19 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: s_mov_b64 exec, s[10:11]
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
@@ -8687,6 +8713,7 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v32, 8
; GCN-O0-NEXT: v_readlane_b32 s5, v32, 9
+; GCN-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
@@ -8704,12 +8731,12 @@ define <8 x double> @double8_inselt_vec(<8 x double> %vec, i32 %sel) {
; GCN-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(14)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v17
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v17
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v16
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
@@ -8867,16 +8894,17 @@ define <3 x i32> @insert_dyn_i32_3(<3 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v5, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v5, 3
+; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v4
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v4
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v3
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
@@ -9008,16 +9036,17 @@ define <3 x float> @insert_dyn_float_3(<3 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v5, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v5, 3
+; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v4
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v4
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v3
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
@@ -9161,18 +9190,19 @@ define <5 x i32> @insert_dyn_i32_5(<5 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v9, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v9, 3
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v6
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v5
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
@@ -9340,18 +9370,19 @@ define <5 x float> @insert_dyn_float_5(<5 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v9, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v9, 3
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(6)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v6
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v5
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
@@ -9525,6 +9556,7 @@ define <6 x i32> @insert_dyn_i32_6(<6 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v11, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v11, 3
+; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -9532,12 +9564,12 @@ define <6 x i32> @insert_dyn_i32_6(<6 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v7
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v7
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v6
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
@@ -9723,6 +9755,7 @@ define <6 x float> @insert_dyn_float_6(<6 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v11, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v11, 3
+; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -9730,12 +9763,12 @@ define <6 x float> @insert_dyn_float_6(<6 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(7)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v7
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v7
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v6
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
@@ -9927,6 +9960,7 @@ define <7 x i32> @insert_dyn_i32_7(<7 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v13, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v13, 3
+; GCN-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -9935,12 +9969,12 @@ define <7 x i32> @insert_dyn_i32_7(<7 x i32> inreg %arg, i32 %idx, i32 %val) {
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(8)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v8
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v8
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v7
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
@@ -10144,6 +10178,7 @@ define <7 x float> @insert_dyn_float_7(<7 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v13, 2
; GCN-O0-NEXT: v_readlane_b32 s5, v13, 3
+; GCN-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -10152,12 +10187,12 @@ define <7 x float> @insert_dyn_float_7(<7 x float> inreg %arg, i32 %idx, float %
; GCN-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
+; GCN-O0-NEXT: s_waitcnt vmcnt(8)
; GCN-O0-NEXT: v_readfirstlane_b32 s6, v8
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], s6, v8
; GCN-O0-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GCN-O0-NEXT: s_mov_b32 m0, s6
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_movreld_b32_e32 v0, v7
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index 124ba142a56f6..243f852004b7e 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -53,13 +53,14 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: s_mov_b32 s7, s8
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_dual_add_f32 v0, v9, v0 :: v_dual_mov_b32 v9, v8
-; SDAG-NEXT: s_waitcnt vmcnt(0)
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-NEXT: s_waitcnt vmcnt(1)
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; SDAG-NEXT: v_add_f32_e32 v0, v1, v0
+; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_add_f32_e32 v0, v2, v0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; SDAG-NEXT: v_dual_mul_f32 v7, 0x3e800000, v0 :: v_dual_mov_b32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 5b95ef59499ac..97b0651565986 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -5621,18 +5621,18 @@ define amdgpu_kernel void @array_3xi16(i8 %arg0, [3 x i16] %arg1) {
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_add_u32 s0, s4, 42
-; VI-NEXT: s_addc_u32 s1, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: s_addc_u32 s1, s5, 0
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: flat_load_ushort v0, v[0:1]
; VI-NEXT: flat_load_ushort v4, v[4:5]
; VI-NEXT: flat_load_ushort v2, v[2:3]
-; VI-NEXT: flat_load_ushort v0, v[0:1]
; VI-NEXT: s_load_dword s0, s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: flat_store_byte v[0:1], v1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: flat_store_short v[0:1], v4
diff --git a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
index 2375e67b7069b..674a5fa3686d0 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
@@ -341,14 +341,15 @@ define amdgpu_kernel void @test_flat_misaligned_v4(ptr %arg) {
; SPLIT-NEXT: v_add_co_u32 v6, vcc_lo, v0, 8
; SPLIT-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
; SPLIT-NEXT: s_clause 0x3
-; SPLIT-NEXT: flat_load_dword v8, v[2:3]
; SPLIT-NEXT: flat_load_dword v9, v[4:5]
; SPLIT-NEXT: flat_load_dword v10, v[0:1]
+; SPLIT-NEXT: flat_load_dword v8, v[2:3]
; SPLIT-NEXT: flat_load_dword v11, v[6:7]
-; SPLIT-NEXT: s_waitcnt vmcnt(2) lgkmcnt(2)
+; SPLIT-NEXT: s_waitcnt vmcnt(3) lgkmcnt(3)
; SPLIT-NEXT: flat_store_dword v[6:7], v9
-; SPLIT-NEXT: s_waitcnt vmcnt(1) lgkmcnt(2)
+; SPLIT-NEXT: s_waitcnt vmcnt(2) lgkmcnt(3)
; SPLIT-NEXT: flat_store_dword v[2:3], v10
+; SPLIT-NEXT: s_waitcnt vmcnt(1) lgkmcnt(3)
; SPLIT-NEXT: flat_store_dword v[0:1], v8
; SPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(3)
; SPLIT-NEXT: flat_store_dword v[4:5], v11
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
index 0e3cf06f89837..407fe02c57f49 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
@@ -821,22 +821,22 @@ define amdgpu_kernel void @fmuladd_v2f16(
; VI-FLUSH-NEXT: s_mov_b32 s14, s10
; VI-FLUSH-NEXT: s_mov_b32 s15, s11
; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; VI-FLUSH-NEXT: s_mov_b32 s12, s2
-; VI-FLUSH-NEXT: s_mov_b32 s13, s3
; VI-FLUSH-NEXT: s_mov_b32 s16, s4
; VI-FLUSH-NEXT: s_mov_b32 s17, s5
-; VI-FLUSH-NEXT: s_mov_b32 s18, s10
-; VI-FLUSH-NEXT: s_mov_b32 s19, s11
; VI-FLUSH-NEXT: s_mov_b32 s4, s6
; VI-FLUSH-NEXT: s_mov_b32 s5, s7
; VI-FLUSH-NEXT: s_mov_b32 s6, s10
; VI-FLUSH-NEXT: s_mov_b32 s7, s11
-; VI-FLUSH-NEXT: buffer_load_dword v0, off, s[12:15], 0
+; VI-FLUSH-NEXT: s_mov_b32 s12, s2
+; VI-FLUSH-NEXT: s_mov_b32 s13, s3
+; VI-FLUSH-NEXT: s_mov_b32 s18, s10
+; VI-FLUSH-NEXT: s_mov_b32 s19, s11
; VI-FLUSH-NEXT: buffer_load_dword v1, off, s[4:7], 0
+; VI-FLUSH-NEXT: buffer_load_dword v0, off, s[12:15], 0
; VI-FLUSH-NEXT: buffer_load_dword v2, off, s[16:19], 0
; VI-FLUSH-NEXT: s_mov_b32 s8, s0
; VI-FLUSH-NEXT: s_mov_b32 s9, s1
-; VI-FLUSH-NEXT: s_waitcnt vmcnt(1)
+; VI-FLUSH-NEXT: s_waitcnt vmcnt(2)
; VI-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: v_mac_f16_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
index 3ed09b1589fb0..9664e98c3be6e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
@@ -2575,9 +2575,9 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX950-NEXT: scratch_load_dword v50, off, s32 offset:84
; GFX950-NEXT: scratch_load_dword v49, off, s32 offset:96
; GFX950-NEXT: scratch_load_dword v48, off, s32 offset:92
-; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: scratch_load_dword v33, off, s32 offset:104
; GFX950-NEXT: scratch_load_dword v32, off, s32 offset:100
+; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: s_waitcnt vmcnt(25)
; GFX950-NEXT: v_max_f64 v[58:59], v[0:1], v[34:35]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[34:35]
@@ -2655,7 +2655,7 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX950-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
; GFX950-NEXT: v_cndmask_b32_e64 v22, v0, 0, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v23, v1, v2, vcc
-; GFX950-NEXT: s_waitcnt vmcnt(6)
+; GFX950-NEXT: s_waitcnt vmcnt(7)
; GFX950-NEXT: v_max_f64 v[0:1], v[24:25], v[32:33]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[24:25], v[32:33]
; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
@@ -2699,25 +2699,25 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24
; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:32
; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:28
-; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:68
-; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:64
-; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:60
-; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:56
-; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:52
+; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
+; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40
; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:48
; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:44
-; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100
-; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96
-; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92
-; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88
-; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:52
+; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:60
+; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:68
+; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:80
; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:76
-; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88
+; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96
+; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92
+; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100
; GFX10-NEXT: s_waitcnt vmcnt(23)
; GFX10-NEXT: v_max_f64 v[82:83], v[0:1], v[31:32]
; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[31:32]
@@ -2728,38 +2728,39 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: v_max_f64 v[32:33], v[4:5], v[35:36]
; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[35:36]
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112
; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112
; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108
; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120
; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116
; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128
; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
-; GFX10-NEXT: s_waitcnt vmcnt(24)
+; GFX10-NEXT: s_waitcnt vmcnt(25)
; GFX10-NEXT: v_max_f64 v[34:35], v[6:7], v[48:49]
; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[48:49]
-; GFX10-NEXT: s_waitcnt vmcnt(21)
-; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[52:53]
-; GFX10-NEXT: s_waitcnt vmcnt(19)
-; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[54:55]
-; GFX10-NEXT: s_waitcnt vmcnt(17)
-; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[64:65]
-; GFX10-NEXT: s_waitcnt vmcnt(16)
+; GFX10-NEXT: s_waitcnt vmcnt(23)
; GFX10-NEXT: v_max_f64 v[48:49], v[8:9], v[37:38]
; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[8:9], v[37:38]
+; GFX10-NEXT: s_waitcnt vmcnt(21)
; GFX10-NEXT: v_max_f64 v[36:37], v[10:11], v[64:65]
+; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[64:65]
+; GFX10-NEXT: s_waitcnt vmcnt(19)
; GFX10-NEXT: v_max_f64 v[38:39], v[12:13], v[54:55]
+; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[54:55]
+; GFX10-NEXT: s_waitcnt vmcnt(17)
; GFX10-NEXT: v_max_f64 v[54:55], v[14:15], v[52:53]
+; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[52:53]
+; GFX10-NEXT: s_waitcnt vmcnt(15)
+; GFX10-NEXT: v_max_f64 v[52:53], v[16:17], v[50:51]
+; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[50:51]
+; GFX10-NEXT: s_waitcnt vmcnt(13)
+; GFX10-NEXT: v_max_f64 v[50:51], v[18:19], v[80:81]
+; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
; GFX10-NEXT: s_waitcnt vmcnt(11)
; GFX10-NEXT: v_max_f64 v[64:65], v[20:21], v[70:71]
; GFX10-NEXT: v_cmp_u_f64_e64 s13, v[20:21], v[70:71]
; GFX10-NEXT: s_waitcnt vmcnt(9)
-; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
-; GFX10-NEXT: s_waitcnt vmcnt(8)
-; GFX10-NEXT: v_max_f64 v[52:53], v[16:17], v[50:51]
-; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[50:51]
-; GFX10-NEXT: v_max_f64 v[50:51], v[18:19], v[80:81]
; GFX10-NEXT: v_max_f64 v[70:71], v[22:23], v[68:69]
; GFX10-NEXT: v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
; GFX10-NEXT: v_cndmask_b32_e64 v6, v34, 0, s6
@@ -2780,7 +2781,7 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: v_cndmask_b32_e64 v21, v65, 0x7ff80000, s13
; GFX10-NEXT: v_cndmask_b32_e64 v22, v70, 0, s14
; GFX10-NEXT: v_cndmask_b32_e64 v23, v71, 0x7ff80000, s14
-; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_max_f64 v[68:69], v[24:25], v[66:67]
; GFX10-NEXT: v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
; GFX10-NEXT: s_waitcnt vmcnt(5)
@@ -2789,10 +2790,10 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_max_f64 v[80:81], v[28:29], v[2:3]
; GFX10-NEXT: v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v82, 0, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_max_f64 v[86:87], v[30:31], v[4:5]
; GFX10-NEXT: v_cmp_u_f64_e64 s18, v[30:31], v[4:5]
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v82, 0, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v1, v83, 0x7ff80000, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v84, 0, s4
; GFX10-NEXT: v_cndmask_b32_e64 v3, v85, 0x7ff80000, s4
@@ -2812,7 +2813,6 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1f
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16
@@ -2843,51 +2843,52 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:108
; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:120
; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:116
+; GFX11-NEXT: scratch_load_b32 v31, off, s32
; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:128
; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:124
-; GFX11-NEXT: s_waitcnt vmcnt(30)
+; GFX11-NEXT: s_waitcnt vmcnt(31)
; GFX11-NEXT: v_max_f64 v[96:97], v[0:1], v[32:33]
; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33]
-; GFX11-NEXT: s_waitcnt vmcnt(28)
+; GFX11-NEXT: s_waitcnt vmcnt(29)
; GFX11-NEXT: v_max_f64 v[32:33], v[2:3], v[34:35]
; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[34:35]
-; GFX11-NEXT: s_waitcnt vmcnt(26)
+; GFX11-NEXT: s_waitcnt vmcnt(27)
; GFX11-NEXT: v_max_f64 v[34:35], v[4:5], v[36:37]
; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[36:37]
-; GFX11-NEXT: s_waitcnt vmcnt(24)
+; GFX11-NEXT: s_waitcnt vmcnt(25)
; GFX11-NEXT: v_max_f64 v[36:37], v[6:7], v[38:39]
; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[38:39]
-; GFX11-NEXT: s_waitcnt vmcnt(22)
+; GFX11-NEXT: s_waitcnt vmcnt(23)
; GFX11-NEXT: v_max_f64 v[38:39], v[8:9], v[48:49]
; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[48:49]
-; GFX11-NEXT: s_waitcnt vmcnt(20)
+; GFX11-NEXT: s_waitcnt vmcnt(21)
; GFX11-NEXT: v_max_f64 v[48:49], v[10:11], v[50:51]
; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[50:51]
-; GFX11-NEXT: s_waitcnt vmcnt(18)
+; GFX11-NEXT: s_waitcnt vmcnt(19)
; GFX11-NEXT: v_max_f64 v[50:51], v[12:13], v[52:53]
; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[52:53]
-; GFX11-NEXT: s_waitcnt vmcnt(16)
+; GFX11-NEXT: s_waitcnt vmcnt(17)
; GFX11-NEXT: v_max_f64 v[52:53], v[14:15], v[54:55]
; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[54:55]
-; GFX11-NEXT: s_waitcnt vmcnt(14)
+; GFX11-NEXT: s_waitcnt vmcnt(15)
; GFX11-NEXT: v_max_f64 v[54:55], v[16:17], v[64:65]
; GFX11-NEXT: v_cmp_u_f64_e64 s7, v[16:17], v[64:65]
-; GFX11-NEXT: s_waitcnt vmcnt(12)
+; GFX11-NEXT: s_waitcnt vmcnt(13)
; GFX11-NEXT: v_max_f64 v[64:65], v[18:19], v[66:67]
; GFX11-NEXT: v_cmp_u_f64_e64 s8, v[18:19], v[66:67]
-; GFX11-NEXT: s_waitcnt vmcnt(10)
+; GFX11-NEXT: s_waitcnt vmcnt(11)
; GFX11-NEXT: v_max_f64 v[66:67], v[20:21], v[68:69]
; GFX11-NEXT: v_cmp_u_f64_e64 s9, v[20:21], v[68:69]
-; GFX11-NEXT: s_waitcnt vmcnt(8)
+; GFX11-NEXT: s_waitcnt vmcnt(9)
; GFX11-NEXT: v_max_f64 v[68:69], v[22:23], v[70:71]
; GFX11-NEXT: v_cmp_u_f64_e64 s10, v[22:23], v[70:71]
-; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: s_waitcnt vmcnt(7)
; GFX11-NEXT: v_max_f64 v[70:71], v[24:25], v[80:81]
; GFX11-NEXT: v_cmp_u_f64_e64 s11, v[24:25], v[80:81]
-; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: s_waitcnt vmcnt(5)
; GFX11-NEXT: v_max_f64 v[80:81], v[26:27], v[82:83]
; GFX11-NEXT: v_cmp_u_f64_e64 s12, v[26:27], v[82:83]
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: v_max_f64 v[82:83], v[28:29], v[84:85]
; GFX11-NEXT: v_cmp_u_f64_e64 s13, v[28:29], v[84:85]
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2935,9 +2936,6 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX1170-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX1170-NEXT: scratch_load_b32 v34, off, s32 offset:12
-; GFX1170-NEXT: scratch_load_b32 v31, off, s32
-; GFX1170-NEXT: scratch_load_b32 v37, off, s32 offset:120
-; GFX1170-NEXT: scratch_load_b32 v39, off, s32 offset:104
; GFX1170-NEXT: scratch_load_b32 v49, off, s32 offset:24
; GFX1170-NEXT: scratch_load_b32 v48, off, s32 offset:20
; GFX1170-NEXT: scratch_load_b32 v51, off, s32 offset:32
@@ -2958,7 +2956,10 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: scratch_load_b32 v80, off, s32 offset:84
; GFX1170-NEXT: scratch_load_b32 v83, off, s32 offset:96
; GFX1170-NEXT: scratch_load_b32 v82, off, s32 offset:92
+; GFX1170-NEXT: scratch_load_b32 v39, off, s32 offset:104
; GFX1170-NEXT: scratch_load_b32 v38, off, s32 offset:100
+; GFX1170-NEXT: scratch_load_b32 v37, off, s32 offset:120
+; GFX1170-NEXT: scratch_load_b32 v31, off, s32
; GFX1170-NEXT: s_waitcnt vmcnt(26)
; GFX1170-NEXT: v_maximum_f64 v[0:1], v[0:1], v[32:33]
; GFX1170-NEXT: s_clause 0x2
@@ -2970,27 +2971,27 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: s_clause 0x1
; GFX1170-NEXT: scratch_load_b32 v35, off, s32 offset:128
; GFX1170-NEXT: scratch_load_b32 v34, off, s32 offset:124
-; GFX1170-NEXT: s_waitcnt vmcnt(24)
+; GFX1170-NEXT: s_waitcnt vmcnt(27)
; GFX1170-NEXT: v_maximum_f64 v[4:5], v[4:5], v[48:49]
-; GFX1170-NEXT: s_waitcnt vmcnt(22)
+; GFX1170-NEXT: s_waitcnt vmcnt(25)
; GFX1170-NEXT: v_maximum_f64 v[6:7], v[6:7], v[50:51]
-; GFX1170-NEXT: s_waitcnt vmcnt(20)
+; GFX1170-NEXT: s_waitcnt vmcnt(23)
; GFX1170-NEXT: v_maximum_f64 v[8:9], v[8:9], v[52:53]
-; GFX1170-NEXT: s_waitcnt vmcnt(18)
+; GFX1170-NEXT: s_waitcnt vmcnt(21)
; GFX1170-NEXT: v_maximum_f64 v[10:11], v[10:11], v[54:55]
-; GFX1170-NEXT: s_waitcnt vmcnt(16)
+; GFX1170-NEXT: s_waitcnt vmcnt(19)
; GFX1170-NEXT: v_maximum_f64 v[12:13], v[12:13], v[64:65]
-; GFX1170-NEXT: s_waitcnt vmcnt(14)
+; GFX1170-NEXT: s_waitcnt vmcnt(17)
; GFX1170-NEXT: v_maximum_f64 v[14:15], v[14:15], v[66:67]
-; GFX1170-NEXT: s_waitcnt vmcnt(12)
+; GFX1170-NEXT: s_waitcnt vmcnt(15)
; GFX1170-NEXT: v_maximum_f64 v[16:17], v[16:17], v[68:69]
-; GFX1170-NEXT: s_waitcnt vmcnt(10)
+; GFX1170-NEXT: s_waitcnt vmcnt(13)
; GFX1170-NEXT: v_maximum_f64 v[18:19], v[18:19], v[70:71]
-; GFX1170-NEXT: s_waitcnt vmcnt(8)
+; GFX1170-NEXT: s_waitcnt vmcnt(11)
; GFX1170-NEXT: v_maximum_f64 v[20:21], v[20:21], v[80:81]
-; GFX1170-NEXT: s_waitcnt vmcnt(6)
+; GFX1170-NEXT: s_waitcnt vmcnt(9)
; GFX1170-NEXT: v_maximum_f64 v[22:23], v[22:23], v[82:83]
-; GFX1170-NEXT: s_waitcnt vmcnt(5)
+; GFX1170-NEXT: s_waitcnt vmcnt(7)
; GFX1170-NEXT: v_maximum_f64 v[24:25], v[24:25], v[38:39]
; GFX1170-NEXT: s_waitcnt vmcnt(3)
; GFX1170-NEXT: v_maximum_f64 v[26:27], v[26:27], v[32:33]
@@ -3008,7 +3009,6 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_clause 0x1f
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
; GFX12-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX12-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX12-NEXT: scratch_load_b32 v35, off, s32 offset:16
@@ -3039,37 +3039,38 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX12-NEXT: scratch_load_b32 v82, off, s32 offset:108
; GFX12-NEXT: scratch_load_b32 v85, off, s32 offset:120
; GFX12-NEXT: scratch_load_b32 v84, off, s32 offset:116
+; GFX12-NEXT: scratch_load_b32 v31, off, s32
; GFX12-NEXT: scratch_load_b32 v87, off, s32 offset:128
; GFX12-NEXT: scratch_load_b32 v86, off, s32 offset:124
-; GFX12-NEXT: s_wait_loadcnt 0x1e
+; GFX12-NEXT: s_wait_loadcnt 0x1f
; GFX12-NEXT: v_maximum_f64 v[0:1], v[0:1], v[32:33]
-; GFX12-NEXT: s_wait_loadcnt 0x1c
+; GFX12-NEXT: s_wait_loadcnt 0x1d
; GFX12-NEXT: v_maximum_f64 v[2:3], v[2:3], v[34:35]
-; GFX12-NEXT: s_wait_loadcnt 0x1a
+; GFX12-NEXT: s_wait_loadcnt 0x1b
; GFX12-NEXT: v_maximum_f64 v[4:5], v[4:5], v[36:37]
-; GFX12-NEXT: s_wait_loadcnt 0x18
+; GFX12-NEXT: s_wait_loadcnt 0x19
; GFX12-NEXT: v_maximum_f64 v[6:7], v[6:7], v[38:39]
-; GFX12-NEXT: s_wait_loadcnt 0x16
+; GFX12-NEXT: s_wait_loadcnt 0x17
; GFX12-NEXT: v_maximum_f64 v[8:9], v[8:9], v[48:49]
-; GFX12-NEXT: s_wait_loadcnt 0x14
+; GFX12-NEXT: s_wait_loadcnt 0x15
; GFX12-NEXT: v_maximum_f64 v[10:11], v[10:11], v[50:51]
-; GFX12-NEXT: s_wait_loadcnt 0x12
+; GFX12-NEXT: s_wait_loadcnt 0x13
; GFX12-NEXT: v_maximum_f64 v[12:13], v[12:13], v[52:53]
-; GFX12-NEXT: s_wait_loadcnt 0x10
+; GFX12-NEXT: s_wait_loadcnt 0x11
; GFX12-NEXT: v_maximum_f64 v[14:15], v[14:15], v[54:55]
-; GFX12-NEXT: s_wait_loadcnt 0xe
+; GFX12-NEXT: s_wait_loadcnt 0xf
; GFX12-NEXT: v_maximum_f64 v[16:17], v[16:17], v[64:65]
-; GFX12-NEXT: s_wait_loadcnt 0xc
+; GFX12-NEXT: s_wait_loadcnt 0xd
; GFX12-NEXT: v_maximum_f64 v[18:19], v[18:19], v[66:67]
-; GFX12-NEXT: s_wait_loadcnt 0xa
+; GFX12-NEXT: s_wait_loadcnt 0xb
; GFX12-NEXT: v_maximum_f64 v[20:21], v[20:21], v[68:69]
-; GFX12-NEXT: s_wait_loadcnt 0x8
+; GFX12-NEXT: s_wait_loadcnt 0x9
; GFX12-NEXT: v_maximum_f64 v[22:23], v[22:23], v[70:71]
-; GFX12-NEXT: s_wait_loadcnt 0x6
+; GFX12-NEXT: s_wait_loadcnt 0x7
; GFX12-NEXT: v_maximum_f64 v[24:25], v[24:25], v[80:81]
-; GFX12-NEXT: s_wait_loadcnt 0x4
+; GFX12-NEXT: s_wait_loadcnt 0x5
; GFX12-NEXT: v_maximum_f64 v[26:27], v[26:27], v[82:83]
-; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: s_wait_loadcnt 0x3
; GFX12-NEXT: v_maximum_f64 v[28:29], v[28:29], v[84:85]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_maximum_f64 v[30:31], v[30:31], v[86:87]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
index 2f18ad05ad3e2..1706f9d5e0250 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
@@ -2575,9 +2575,9 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX950-NEXT: scratch_load_dword v50, off, s32 offset:84
; GFX950-NEXT: scratch_load_dword v49, off, s32 offset:96
; GFX950-NEXT: scratch_load_dword v48, off, s32 offset:92
-; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: scratch_load_dword v33, off, s32 offset:104
; GFX950-NEXT: scratch_load_dword v32, off, s32 offset:100
+; GFX950-NEXT: scratch_load_dword v31, off, s32
; GFX950-NEXT: s_waitcnt vmcnt(25)
; GFX950-NEXT: v_min_f64 v[58:59], v[0:1], v[34:35]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[34:35]
@@ -2655,7 +2655,7 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX950-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
; GFX950-NEXT: v_cndmask_b32_e64 v22, v0, 0, vcc
; GFX950-NEXT: v_cndmask_b32_e32 v23, v1, v2, vcc
-; GFX950-NEXT: s_waitcnt vmcnt(6)
+; GFX950-NEXT: s_waitcnt vmcnt(7)
; GFX950-NEXT: v_min_f64 v[0:1], v[24:25], v[32:33]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[24:25], v[32:33]
; GFX950-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
@@ -2699,25 +2699,25 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:12
; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:24
; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:20
-; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:32
; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:28
-; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:68
-; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:64
-; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:60
-; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:56
-; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:52
+; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:36
+; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40
; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:48
; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:44
-; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:40
-; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100
-; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96
-; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92
-; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88
-; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:56
+; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:52
+; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:64
+; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:60
+; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:68
+; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
; GFX10-NEXT: buffer_load_dword v81, off, s[0:3], s32 offset:80
; GFX10-NEXT: buffer_load_dword v80, off, s[0:3], s32 offset:76
-; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:72
+; GFX10-NEXT: buffer_load_dword v71, off, s[0:3], s32 offset:88
+; GFX10-NEXT: buffer_load_dword v70, off, s[0:3], s32 offset:84
+; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:96
+; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:92
+; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:100
; GFX10-NEXT: s_waitcnt vmcnt(23)
; GFX10-NEXT: v_min_f64 v[82:83], v[0:1], v[31:32]
; GFX10-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[31:32]
@@ -2728,38 +2728,39 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: v_min_f64 v[32:33], v[4:5], v[35:36]
; GFX10-NEXT: v_cmp_u_f64_e64 s5, v[4:5], v[35:36]
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112
; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:104
+; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112
; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108
; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120
; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116
; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128
; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124
-; GFX10-NEXT: s_waitcnt vmcnt(24)
+; GFX10-NEXT: s_waitcnt vmcnt(25)
; GFX10-NEXT: v_min_f64 v[34:35], v[6:7], v[48:49]
; GFX10-NEXT: v_cmp_u_f64_e64 s6, v[6:7], v[48:49]
-; GFX10-NEXT: s_waitcnt vmcnt(21)
-; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[52:53]
-; GFX10-NEXT: s_waitcnt vmcnt(19)
-; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[54:55]
-; GFX10-NEXT: s_waitcnt vmcnt(17)
-; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[64:65]
-; GFX10-NEXT: s_waitcnt vmcnt(16)
+; GFX10-NEXT: s_waitcnt vmcnt(23)
; GFX10-NEXT: v_min_f64 v[48:49], v[8:9], v[37:38]
; GFX10-NEXT: v_cmp_u_f64_e64 s7, v[8:9], v[37:38]
+; GFX10-NEXT: s_waitcnt vmcnt(21)
; GFX10-NEXT: v_min_f64 v[36:37], v[10:11], v[64:65]
+; GFX10-NEXT: v_cmp_u_f64_e64 s8, v[10:11], v[64:65]
+; GFX10-NEXT: s_waitcnt vmcnt(19)
; GFX10-NEXT: v_min_f64 v[38:39], v[12:13], v[54:55]
+; GFX10-NEXT: v_cmp_u_f64_e64 s9, v[12:13], v[54:55]
+; GFX10-NEXT: s_waitcnt vmcnt(17)
; GFX10-NEXT: v_min_f64 v[54:55], v[14:15], v[52:53]
+; GFX10-NEXT: v_cmp_u_f64_e64 s10, v[14:15], v[52:53]
+; GFX10-NEXT: s_waitcnt vmcnt(15)
+; GFX10-NEXT: v_min_f64 v[52:53], v[16:17], v[50:51]
+; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[50:51]
+; GFX10-NEXT: s_waitcnt vmcnt(13)
+; GFX10-NEXT: v_min_f64 v[50:51], v[18:19], v[80:81]
+; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
; GFX10-NEXT: s_waitcnt vmcnt(11)
; GFX10-NEXT: v_min_f64 v[64:65], v[20:21], v[70:71]
; GFX10-NEXT: v_cmp_u_f64_e64 s13, v[20:21], v[70:71]
; GFX10-NEXT: s_waitcnt vmcnt(9)
-; GFX10-NEXT: v_cmp_u_f64_e64 s12, v[18:19], v[80:81]
-; GFX10-NEXT: s_waitcnt vmcnt(8)
-; GFX10-NEXT: v_min_f64 v[52:53], v[16:17], v[50:51]
-; GFX10-NEXT: v_cmp_u_f64_e64 s11, v[16:17], v[50:51]
-; GFX10-NEXT: v_min_f64 v[50:51], v[18:19], v[80:81]
; GFX10-NEXT: v_min_f64 v[70:71], v[22:23], v[68:69]
; GFX10-NEXT: v_cmp_u_f64_e64 s14, v[22:23], v[68:69]
; GFX10-NEXT: v_cndmask_b32_e64 v6, v34, 0, s6
@@ -2780,7 +2781,7 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: v_cndmask_b32_e64 v21, v65, 0x7ff80000, s13
; GFX10-NEXT: v_cndmask_b32_e64 v22, v70, 0, s14
; GFX10-NEXT: v_cndmask_b32_e64 v23, v71, 0x7ff80000, s14
-; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_min_f64 v[68:69], v[24:25], v[66:67]
; GFX10-NEXT: v_cmp_u_f64_e64 s15, v[24:25], v[66:67]
; GFX10-NEXT: s_waitcnt vmcnt(5)
@@ -2789,10 +2790,10 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_min_f64 v[80:81], v[28:29], v[2:3]
; GFX10-NEXT: v_cmp_u_f64_e64 s17, v[28:29], v[2:3]
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v82, 0, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_min_f64 v[86:87], v[30:31], v[4:5]
; GFX10-NEXT: v_cmp_u_f64_e64 s18, v[30:31], v[4:5]
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v82, 0, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v1, v83, 0x7ff80000, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v84, 0, s4
; GFX10-NEXT: v_cndmask_b32_e64 v3, v85, 0x7ff80000, s4
@@ -2812,7 +2813,6 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1f
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16
@@ -2843,51 +2843,52 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:108
; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:120
; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:116
+; GFX11-NEXT: scratch_load_b32 v31, off, s32
; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:128
; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:124
-; GFX11-NEXT: s_waitcnt vmcnt(30)
+; GFX11-NEXT: s_waitcnt vmcnt(31)
; GFX11-NEXT: v_min_f64 v[96:97], v[0:1], v[32:33]
; GFX11-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[32:33]
-; GFX11-NEXT: s_waitcnt vmcnt(28)
+; GFX11-NEXT: s_waitcnt vmcnt(29)
; GFX11-NEXT: v_min_f64 v[32:33], v[2:3], v[34:35]
; GFX11-NEXT: v_cmp_u_f64_e64 s0, v[2:3], v[34:35]
-; GFX11-NEXT: s_waitcnt vmcnt(26)
+; GFX11-NEXT: s_waitcnt vmcnt(27)
; GFX11-NEXT: v_min_f64 v[34:35], v[4:5], v[36:37]
; GFX11-NEXT: v_cmp_u_f64_e64 s1, v[4:5], v[36:37]
-; GFX11-NEXT: s_waitcnt vmcnt(24)
+; GFX11-NEXT: s_waitcnt vmcnt(25)
; GFX11-NEXT: v_min_f64 v[36:37], v[6:7], v[38:39]
; GFX11-NEXT: v_cmp_u_f64_e64 s2, v[6:7], v[38:39]
-; GFX11-NEXT: s_waitcnt vmcnt(22)
+; GFX11-NEXT: s_waitcnt vmcnt(23)
; GFX11-NEXT: v_min_f64 v[38:39], v[8:9], v[48:49]
; GFX11-NEXT: v_cmp_u_f64_e64 s3, v[8:9], v[48:49]
-; GFX11-NEXT: s_waitcnt vmcnt(20)
+; GFX11-NEXT: s_waitcnt vmcnt(21)
; GFX11-NEXT: v_min_f64 v[48:49], v[10:11], v[50:51]
; GFX11-NEXT: v_cmp_u_f64_e64 s4, v[10:11], v[50:51]
-; GFX11-NEXT: s_waitcnt vmcnt(18)
+; GFX11-NEXT: s_waitcnt vmcnt(19)
; GFX11-NEXT: v_min_f64 v[50:51], v[12:13], v[52:53]
; GFX11-NEXT: v_cmp_u_f64_e64 s5, v[12:13], v[52:53]
-; GFX11-NEXT: s_waitcnt vmcnt(16)
+; GFX11-NEXT: s_waitcnt vmcnt(17)
; GFX11-NEXT: v_min_f64 v[52:53], v[14:15], v[54:55]
; GFX11-NEXT: v_cmp_u_f64_e64 s6, v[14:15], v[54:55]
-; GFX11-NEXT: s_waitcnt vmcnt(14)
+; GFX11-NEXT: s_waitcnt vmcnt(15)
; GFX11-NEXT: v_min_f64 v[54:55], v[16:17], v[64:65]
; GFX11-NEXT: v_cmp_u_f64_e64 s7, v[16:17], v[64:65]
-; GFX11-NEXT: s_waitcnt vmcnt(12)
+; GFX11-NEXT: s_waitcnt vmcnt(13)
; GFX11-NEXT: v_min_f64 v[64:65], v[18:19], v[66:67]
; GFX11-NEXT: v_cmp_u_f64_e64 s8, v[18:19], v[66:67]
-; GFX11-NEXT: s_waitcnt vmcnt(10)
+; GFX11-NEXT: s_waitcnt vmcnt(11)
; GFX11-NEXT: v_min_f64 v[66:67], v[20:21], v[68:69]
; GFX11-NEXT: v_cmp_u_f64_e64 s9, v[20:21], v[68:69]
-; GFX11-NEXT: s_waitcnt vmcnt(8)
+; GFX11-NEXT: s_waitcnt vmcnt(9)
; GFX11-NEXT: v_min_f64 v[68:69], v[22:23], v[70:71]
; GFX11-NEXT: v_cmp_u_f64_e64 s10, v[22:23], v[70:71]
-; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: s_waitcnt vmcnt(7)
; GFX11-NEXT: v_min_f64 v[70:71], v[24:25], v[80:81]
; GFX11-NEXT: v_cmp_u_f64_e64 s11, v[24:25], v[80:81]
-; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: s_waitcnt vmcnt(5)
; GFX11-NEXT: v_min_f64 v[80:81], v[26:27], v[82:83]
; GFX11-NEXT: v_cmp_u_f64_e64 s12, v[26:27], v[82:83]
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: v_min_f64 v[82:83], v[28:29], v[84:85]
; GFX11-NEXT: v_cmp_u_f64_e64 s13, v[28:29], v[84:85]
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2935,9 +2936,6 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX1170-NEXT: scratch_load_b32 v35, off, s32 offset:16
; GFX1170-NEXT: scratch_load_b32 v34, off, s32 offset:12
-; GFX1170-NEXT: scratch_load_b32 v31, off, s32
-; GFX1170-NEXT: scratch_load_b32 v37, off, s32 offset:120
-; GFX1170-NEXT: scratch_load_b32 v39, off, s32 offset:104
; GFX1170-NEXT: scratch_load_b32 v49, off, s32 offset:24
; GFX1170-NEXT: scratch_load_b32 v48, off, s32 offset:20
; GFX1170-NEXT: scratch_load_b32 v51, off, s32 offset:32
@@ -2958,7 +2956,10 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: scratch_load_b32 v80, off, s32 offset:84
; GFX1170-NEXT: scratch_load_b32 v83, off, s32 offset:96
; GFX1170-NEXT: scratch_load_b32 v82, off, s32 offset:92
+; GFX1170-NEXT: scratch_load_b32 v39, off, s32 offset:104
; GFX1170-NEXT: scratch_load_b32 v38, off, s32 offset:100
+; GFX1170-NEXT: scratch_load_b32 v37, off, s32 offset:120
+; GFX1170-NEXT: scratch_load_b32 v31, off, s32
; GFX1170-NEXT: s_waitcnt vmcnt(26)
; GFX1170-NEXT: v_minimum_f64 v[0:1], v[0:1], v[32:33]
; GFX1170-NEXT: s_clause 0x2
@@ -2970,27 +2971,27 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX1170-NEXT: s_clause 0x1
; GFX1170-NEXT: scratch_load_b32 v35, off, s32 offset:128
; GFX1170-NEXT: scratch_load_b32 v34, off, s32 offset:124
-; GFX1170-NEXT: s_waitcnt vmcnt(24)
+; GFX1170-NEXT: s_waitcnt vmcnt(27)
; GFX1170-NEXT: v_minimum_f64 v[4:5], v[4:5], v[48:49]
-; GFX1170-NEXT: s_waitcnt vmcnt(22)
+; GFX1170-NEXT: s_waitcnt vmcnt(25)
; GFX1170-NEXT: v_minimum_f64 v[6:7], v[6:7], v[50:51]
-; GFX1170-NEXT: s_waitcnt vmcnt(20)
+; GFX1170-NEXT: s_waitcnt vmcnt(23)
; GFX1170-NEXT: v_minimum_f64 v[8:9], v[8:9], v[52:53]
-; GFX1170-NEXT: s_waitcnt vmcnt(18)
+; GFX1170-NEXT: s_waitcnt vmcnt(21)
; GFX1170-NEXT: v_minimum_f64 v[10:11], v[10:11], v[54:55]
-; GFX1170-NEXT: s_waitcnt vmcnt(16)
+; GFX1170-NEXT: s_waitcnt vmcnt(19)
; GFX1170-NEXT: v_minimum_f64 v[12:13], v[12:13], v[64:65]
-; GFX1170-NEXT: s_waitcnt vmcnt(14)
+; GFX1170-NEXT: s_waitcnt vmcnt(17)
; GFX1170-NEXT: v_minimum_f64 v[14:15], v[14:15], v[66:67]
-; GFX1170-NEXT: s_waitcnt vmcnt(12)
+; GFX1170-NEXT: s_waitcnt vmcnt(15)
; GFX1170-NEXT: v_minimum_f64 v[16:17], v[16:17], v[68:69]
-; GFX1170-NEXT: s_waitcnt vmcnt(10)
+; GFX1170-NEXT: s_waitcnt vmcnt(13)
; GFX1170-NEXT: v_minimum_f64 v[18:19], v[18:19], v[70:71]
-; GFX1170-NEXT: s_waitcnt vmcnt(8)
+; GFX1170-NEXT: s_waitcnt vmcnt(11)
; GFX1170-NEXT: v_minimum_f64 v[20:21], v[20:21], v[80:81]
-; GFX1170-NEXT: s_waitcnt vmcnt(6)
+; GFX1170-NEXT: s_waitcnt vmcnt(9)
; GFX1170-NEXT: v_minimum_f64 v[22:23], v[22:23], v[82:83]
-; GFX1170-NEXT: s_waitcnt vmcnt(5)
+; GFX1170-NEXT: s_waitcnt vmcnt(7)
; GFX1170-NEXT: v_minimum_f64 v[24:25], v[24:25], v[38:39]
; GFX1170-NEXT: s_waitcnt vmcnt(3)
; GFX1170-NEXT: v_minimum_f64 v[26:27], v[26:27], v[32:33]
@@ -3008,7 +3009,6 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_clause 0x1f
-; GFX12-NEXT: scratch_load_b32 v31, off, s32
; GFX12-NEXT: scratch_load_b32 v33, off, s32 offset:8
; GFX12-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX12-NEXT: scratch_load_b32 v35, off, s32 offset:16
@@ -3039,37 +3039,38 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
; GFX12-NEXT: scratch_load_b32 v82, off, s32 offset:108
; GFX12-NEXT: scratch_load_b32 v85, off, s32 offset:120
; GFX12-NEXT: scratch_load_b32 v84, off, s32 offset:116
+; GFX12-NEXT: scratch_load_b32 v31, off, s32
; GFX12-NEXT: scratch_load_b32 v87, off, s32 offset:128
; GFX12-NEXT: scratch_load_b32 v86, off, s32 offset:124
-; GFX12-NEXT: s_wait_loadcnt 0x1e
+; GFX12-NEXT: s_wait_loadcnt 0x1f
; GFX12-NEXT: v_minimum_f64 v[0:1], v[0:1], v[32:33]
-; GFX12-NEXT: s_wait_loadcnt 0x1c
+; GFX12-NEXT: s_wait_loadcnt 0x1d
; GFX12-NEXT: v_minimum_f64 v[2:3], v[2:3], v[34:35]
-; GFX12-NEXT: s_wait_loadcnt 0x1a
+; GFX12-NEXT: s_wait_loadcnt 0x1b
; GFX12-NEXT: v_minimum_f64 v[4:5], v[4:5], v[36:37]
-; GFX12-NEXT: s_wait_loadcnt 0x18
+; GFX12-NEXT: s_wait_loadcnt 0x19
; GFX12-NEXT: v_minimum_f64 v[6:7], v[6:7], v[38:39]
-; GFX12-NEXT: s_wait_loadcnt 0x16
+; GFX12-NEXT: s_wait_loadcnt 0x17
; GFX12-NEXT: v_minimum_f64 v[8:9], v[8:9], v[48:49]
-; GFX12-NEXT: s_wait_loadcnt 0x14
+; GFX12-NEXT: s_wait_loadcnt 0x15
; GFX12-NEXT: v_minimum_f64 v[10:11], v[10:11], v[50:51]
-; GFX12-NEXT: s_wait_loadcnt 0x12
+; GFX12-NEXT: s_wait_loadcnt 0x13
; GFX12-NEXT: v_minimum_f64 v[12:13], v[12:13], v[52:53]
-; GFX12-NEXT: s_wait_loadcnt 0x10
+; GFX12-NEXT: s_wait_loadcnt 0x11
; GFX12-NEXT: v_minimum_f64 v[14:15], v[14:15], v[54:55]
-; GFX12-NEXT: s_wait_loadcnt 0xe
+; GFX12-NEXT: s_wait_loadcnt 0xf
; GFX12-NEXT: v_minimum_f64 v[16:17], v[16:17], v[64:65]
-; GFX12-NEXT: s_wait_loadcnt 0xc
+; GFX12-NEXT: s_wait_loadcnt 0xd
; GFX12-NEXT: v_minimum_f64 v[18:19], v[18:19], v[66:67]
-; GFX12-NEXT: s_wait_loadcnt 0xa
+; GFX12-NEXT: s_wait_loadcnt 0xb
; GFX12-NEXT: v_minimum_f64 v[20:21], v[20:21], v[68:69]
-; GFX12-NEXT: s_wait_loadcnt 0x8
+; GFX12-NEXT: s_wait_loadcnt 0x9
; GFX12-NEXT: v_minimum_f64 v[22:23], v[22:23], v[70:71]
-; GFX12-NEXT: s_wait_loadcnt 0x6
+; GFX12-NEXT: s_wait_loadcnt 0x7
; GFX12-NEXT: v_minimum_f64 v[24:25], v[24:25], v[80:81]
-; GFX12-NEXT: s_wait_loadcnt 0x4
+; GFX12-NEXT: s_wait_loadcnt 0x5
; GFX12-NEXT: v_minimum_f64 v[26:27], v[26:27], v[82:83]
-; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: s_wait_loadcnt 0x3
; GFX12-NEXT: v_minimum_f64 v[28:29], v[28:29], v[84:85]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_minimum_f64 v[30:31], v[30:31], v[86:87]
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 1536ac5112e7b..7281476b63f36 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -773,25 +773,24 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-TRUE16-NEXT: global_load_d16_b16 v2, v8, s[0:1] offset:8
; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v8, s[0:1] offset:4
; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v8, s[0:1]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x6
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x6
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x6
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x6
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x5
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x5
+; GFX12-TRUE16-NEXT: s_clause 0x2
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: s_clause 0x1
+; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
+; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x2
; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[4:7], off
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[0:3], off
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -809,25 +808,24 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-FAKE16-NEXT: global_load_u16 v2, v8, s[0:1] offset:8
; GFX12-FAKE16-NEXT: global_load_u16 v1, v8, s[0:1] offset:4
; GFX12-FAKE16-NEXT: global_load_u16 v0, v8, s[0:1]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x6
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x6
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x6
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x6
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x5
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x5
+; GFX12-FAKE16-NEXT: s_clause 0x2
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: s_clause 0x1
+; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
+; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x2
; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[4:7], off
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[0:3], off
; GFX12-FAKE16-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
index 8d7eb6dcb843c..ed48db59d392f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-f32.ll
@@ -712,8 +712,8 @@ define amdgpu_kernel void @global_load_v16f32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCN-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GCN-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCN-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCN-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCN-HSA-NEXT: v_mov_b32_e32 v13, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v12, s4
; GCN-HSA-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
@@ -734,8 +734,9 @@ define amdgpu_kernel void @global_load_v16f32(ptr addrspace(1) %out, ptr addrspa
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v3, s1
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s0
-; GCN-HSA-NEXT: s_waitcnt vmcnt(2)
+; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
; GCN-HSA-NEXT: flat_store_dwordx4 v[16:17], v[8:11]
+; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
; GCN-HSA-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; GCN-HSA-NEXT: s_waitcnt vmcnt(3)
; GCN-HSA-NEXT: flat_store_dwordx4 v[2:3], v[12:15]
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
index 930fe409c2cea..b6ea8f2335b2c 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -3929,8 +3929,8 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s7
; GCNX3-HSA-NEXT: v_mov_b32_e32 v8, s6
-; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCNX3-HSA-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GCNX3-HSA-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GCNX3-HSA-NEXT: v_mov_b32_e32 v1, 0
; GCNX3-HSA-NEXT: s_add_u32 s2, s0, 16
; GCNX3-HSA-NEXT: v_mov_b32_e32 v3, v1
@@ -4024,7 +4024,7 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v14
; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v15
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
-; GCNX3-HSA-NEXT: s_waitcnt vmcnt(12)
+; GCNX3-HSA-NEXT: s_waitcnt vmcnt(13)
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v8
; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v9
; GCNX3-HSA-NEXT: v_mov_b32_e32 v9, s3
@@ -4041,6 +4041,7 @@ define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(ptr addrspace(1) %ou
; GCNX3-HSA-NEXT: s_add_u32 s0, s0, 48
; GCNX3-HSA-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GCNX3-HSA-NEXT: s_addc_u32 s1, s1, 0
+; GCNX3-HSA-NEXT: s_waitcnt vmcnt(14)
; GCNX3-HSA-NEXT: v_mov_b32_e32 v0, v4
; GCNX3-HSA-NEXT: v_mov_b32_e32 v2, v5
; GCNX3-HSA-NEXT: v_mov_b32_e32 v5, s3
@@ -4361,21 +4362,23 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; SI-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; SI-NOHSA-NEXT: s_mov_b32 s8, s2
; SI-NOHSA-NEXT: s_mov_b32 s9, s3
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:96
-; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:64
+; SI-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:32
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0
; SI-NOHSA-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:16
; SI-NOHSA-NEXT: s_mov_b32 s4, s0
; SI-NOHSA-NEXT: s_mov_b32 s5, s1
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(6)
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:96
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:112
-; SI-NOHSA-NEXT: s_waitcnt vmcnt(6)
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:64
+; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:80
; SI-NOHSA-NEXT: s_waitcnt vmcnt(7)
; SI-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 offset:32
@@ -4491,21 +4494,23 @@ define amdgpu_kernel void @global_load_v32i32(ptr addrspace(1) %out, ptr addrspa
; GCNX3-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GCNX3-NOHSA-NEXT: s_mov_b32 s8, s2
; GCNX3-NOHSA-NEXT: s_mov_b32 s9, s3
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:96
-; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0 offset:112
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[12:15], off, s[8:11], 0 offset:64
+; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[8:11], off, s[8:11], 0 offset:80
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[16:19], off, s[8:11], 0 offset:32
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[20:23], off, s[8:11], 0 offset:48
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[24:27], off, s[8:11], 0
; GCNX3-NOHSA-NEXT: buffer_load_dwordx4 v[28:31], off, s[8:11], 0 offset:16
; GCNX3-NOHSA-NEXT: s_mov_b32 s4, s0
; GCNX3-NOHSA-NEXT: s_mov_b32 s5, s1
-; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(6)
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:96
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 offset:112
-; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(6)
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 offset:64
+; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 offset:80
; GCNX3-NOHSA-NEXT: s_waitcnt vmcnt(7)
; GCNX3-NOHSA-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index 43196c3d08440..2b8474e75b53f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -7268,10 +7268,10 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI-NEXT: v_mov_b32_e32 v19, v4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v12, s1
-; SI-NEXT: ds_read2_b64 v[0:3], v12 offset1:1
; SI-NEXT: ds_read2_b64 v[5:8], v12 offset0:2 offset1:3
+; SI-NEXT: ds_read2_b64 v[0:3], v12 offset1:1
; SI-NEXT: v_mov_b32_e32 v20, s0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_waitcnt lgkmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v8
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v8
; SI-NEXT: ds_read2_b64 v[8:11], v12 offset0:4 offset1:5
@@ -7280,6 +7280,7 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v6
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v6
; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:10 offset1:11
+; SI-NEXT: s_waitcnt lgkmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v3
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v3
; SI-NEXT: ds_write2_b64 v20, v[18:19], v[16:17] offset0:6 offset1:7
@@ -7425,12 +7426,12 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v21, v5
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v22, s0
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset1:1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset1:1
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v2
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v2
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[20:21] offset0:28 offset1:29
@@ -7441,6 +7442,7 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v0
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v0
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[1:2], v[18:19] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v9
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v9
@@ -7457,7 +7459,7 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v5
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:16 offset1:17
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(7)
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v17
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v17
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:14 offset1:15
@@ -7469,6 +7471,7 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:10 offset1:11
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v14
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v14
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(10)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v10
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:8 offset1:9
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index cd0485fa1ab15..7ca14a5d1e0b8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -11,13 +11,14 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
; CHECK-NEXT: v_add_i32_e32 v2, vcc, 8, v0
; CHECK-NEXT: s_mov_b32 m0, -1
; CHECK-NEXT: v_add_i32_e32 v4, vcc, 4, v0
+; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: ds_read_b32 v3, v1
; CHECK-NEXT: ds_read_b32 v2, v2
; CHECK-NEXT: ds_read_b32 v1, v4
-; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: v_mov_b32_e32 v4, 0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_waitcnt lgkmcnt(3)
; CHECK-NEXT: exp mrt0, off, off, off, off
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: tbuffer_store_format_xyzw v[0:3], v4, s[0:3], 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_FLOAT] idxen
; CHECK-NEXT: s_endpgm
call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float poison, float poison, float poison, float poison, i1 false, i1 false)
@@ -38,12 +39,12 @@ define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg
; CHECK-NEXT: v_add_i32_e32 v8, vcc, 8, v1
; CHECK-NEXT: s_mov_b32 m0, -1
; CHECK-NEXT: v_add_i32_e32 v10, vcc, 4, v1
-; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: ds_read_b32 v5, v4
-; CHECK-NEXT: ds_read_b32 v4, v6
; CHECK-NEXT: ds_read_b32 v9, v7
; CHECK-NEXT: ds_read_b32 v8, v8
; CHECK-NEXT: ds_read_b32 v7, v10
+; CHECK-NEXT: ds_read_b32 v2, v2
+; CHECK-NEXT: ds_read_b32 v5, v4
+; CHECK-NEXT: ds_read_b32 v4, v6
; CHECK-NEXT: ds_read_b32 v6, v1
; CHECK-NEXT: ds_read_b32 v3, v3
; CHECK-NEXT: s_waitcnt lgkmcnt(1)
@@ -71,16 +72,17 @@ define amdgpu_vs void @test_3(i32 inreg %arg1, i32 inreg %arg2, ptr addrspace(8)
; CHECK-NEXT: ds_read_b32 v6, v0
; CHECK-NEXT: ds_read_b32 v5, v3
; CHECK-NEXT: ds_read_b32 v4, v4
+; CHECK-NEXT: ds_read_b32 v3, v1
; CHECK-NEXT: ds_read_b32 v8, v7
; CHECK-NEXT: ds_read_b32 v7, v9
-; CHECK-NEXT: ds_read_b32 v3, v1
; CHECK-NEXT: s_mov_b32 s7, s5
; CHECK-NEXT: s_mov_b32 s6, s4
; CHECK-NEXT: s_mov_b32 s5, s3
; CHECK-NEXT: s_mov_b32 s4, s2
; CHECK-NEXT: v_mov_b32_e32 v9, s0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_waitcnt lgkmcnt(2)
; CHECK-NEXT: tbuffer_store_format_xyzw v[3:6], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:264 glc slc
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: tbuffer_store_format_xy v[7:8], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:280 glc slc
; CHECK-NEXT: v_add_i32_e32 v0, vcc, 12, v2
; CHECK-NEXT: v_add_i32_e32 v1, vcc, 8, v2
@@ -90,15 +92,17 @@ define amdgpu_vs void @test_3(i32 inreg %arg1, i32 inreg %arg2, ptr addrspace(8)
; CHECK-NEXT: s_waitcnt expcnt(0)
; CHECK-NEXT: v_add_i32_e32 v7, vcc, 16, v2
; CHECK-NEXT: ds_read_b32 v5, v0
+; CHECK-NEXT: ds_read_b32 v0, v7
; CHECK-NEXT: ds_read_b32 v4, v1
; CHECK-NEXT: ds_read_b32 v3, v3
-; CHECK-NEXT: ds_read_b32 v1, v6
-; CHECK-NEXT: ds_read_b32 v0, v7
; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: ds_read_b32 v1, v6
+; CHECK-NEXT: s_waitcnt lgkmcnt(4)
+; CHECK-NEXT: exp mrt0, off, off, off, off
+; CHECK-NEXT: s_waitcnt lgkmcnt(1)
; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:240 glc slc
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: tbuffer_store_format_xy v[0:1], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
-; CHECK-NEXT: exp mrt0, off, off, off, off
; CHECK-NEXT: s_endpgm
%load1 = load <6 x float>, ptr addrspace(3) %arg5, align 4
%vec11 = shufflevector <6 x float> %load1, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AMDGPU/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
index d74af4e792f57..d2165869c0518 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
@@ -96,35 +96,38 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
; GFX7-NEXT: ds_read_u8 v5, v0
; GFX7-NEXT: ds_read_u8 v6, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
; GFX7-NEXT: ds_read_u8 v7, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_or_b32_e32 v4, v4, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v7
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
; GFX7-NEXT: ds_read_u8 v5, v0 offset:9
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
+; GFX7-NEXT: ds_read_u8 v11, v0 offset:10
; GFX7-NEXT: ds_read_u8 v7, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v10, v0 offset:12
; GFX7-NEXT: ds_read_u8 v8, v0 offset:15
; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
-; GFX7-NEXT: ds_read_u8 v10, v0 offset:12
-; GFX7-NEXT: ds_read_u8 v11, v0 offset:10
; GFX7-NEXT: ds_read_u8 v0, v0 offset:8
+; GFX7-NEXT: s_waitcnt lgkmcnt(8)
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
@@ -332,25 +335,27 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:12
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v4, v0 offset:4
; GFX7-NEXT: ds_read_u16 v5, v0
; GFX7-NEXT: ds_read_u16 v6, v0 offset:6
+; GFX7-NEXT: ds_read_u16 v4, v0 offset:4
; GFX7-NEXT: ds_read_u16 v7, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
; GFX7-NEXT: ds_read_u16 v8, v0 offset:14
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:12
; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(3)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/load-local.96.ll b/llvm/test/CodeGen/AMDGPU/load-local.96.ll
index bf70cf42b23af..99591b8a105af 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local.96.ll
@@ -87,31 +87,34 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v5, v0
; GFX7-NEXT: ds_read_u8 v6, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
; GFX7-NEXT: ds_read_u8 v7, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:4
; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
+; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
; GFX7-NEXT: s_waitcnt lgkmcnt(7)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_or_b32_e32 v3, v3, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v7
; GFX7-NEXT: ds_read_u8 v5, v0 offset:9
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
; GFX7-NEXT: ds_read_u8 v0, v0 offset:8
+; GFX7-NEXT: s_waitcnt lgkmcnt(6)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v8
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
@@ -275,20 +278,21 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
; GFX7-NEXT: ds_read_u16 v4, v0
; GFX7-NEXT: ds_read_u16 v5, v0 offset:6
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
; GFX7-NEXT: s_waitcnt lgkmcnt(5)
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/max.i16.ll b/llvm/test/CodeGen/AMDGPU/max.i16.ll
index de6aa1e69bd3a..5ec66fdf5c84d 100644
--- a/llvm/test/CodeGen/AMDGPU/max.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.i16.ll
@@ -116,9 +116,9 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(ptr addrspace(1) %out, ptr addr
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT: flat_load_dword v7, v[2:3]
; VI-NEXT: flat_load_ushort v4, v[4:5]
; VI-NEXT: flat_load_dword v5, v[0:1]
-; VI-NEXT: flat_load_dword v7, v[2:3]
; VI-NEXT: v_add_u32_e32 v0, vcc, 4, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v8, v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 4b16b61f50ecc..6233f4b4bbe6d 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -186,14 +186,14 @@ define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %
; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72
; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68
; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48
; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52
; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56
; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v25, s1
; CHECK-NEXT: v_mov_b32_e32 v24, s0
@@ -201,14 +201,14 @@ define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
+; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80
@@ -474,14 +474,14 @@ define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %
; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72
; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68
; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48
; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52
; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56
; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60
+; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v25, s1
; CHECK-NEXT: v_mov_b32_e32 v24, s0
@@ -489,14 +489,14 @@ define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
+; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
index dacde84ba4551..79912c529266b 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-param-combinations.ll
@@ -844,16 +844,17 @@ define void @memcpy_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[7:8] offset:23
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[9:10] offset:16
@@ -869,15 +870,15 @@ define void @memcpy_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addr
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
@@ -911,16 +912,17 @@ define void @memcpy_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[7:8] offset:23
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[9:10] offset:16
@@ -936,15 +938,15 @@ define void @memcpy_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addr
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index 61f79ccd55f1c..73f150899b999 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -111,10 +111,11 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: flat_load_dwordx4 v[20:23], v[4:5] offset:128
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v5, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
-; ALIGNED-NEXT: flat_load_dwordx4 v[26:29], v[4:5]
-; ALIGNED-NEXT: flat_load_dwordx4 v[35:38], v[4:5] offset:16
-; ALIGNED-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[48:51], v[4:5] offset:64
+; ALIGNED-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[35:38], v[4:5] offset:16
+; ALIGNED-NEXT: flat_load_dwordx4 v[26:29], v[4:5]
+; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[24:25] offset:192
; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[24:25]
; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[24:25] offset:48
@@ -125,7 +126,6 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[24:25] offset:144
; ALIGNED-NEXT: flat_load_dwordx4 v[55:58], v[24:25] offset:160
; ALIGNED-NEXT: flat_load_dwordx4 v[59:62], v[24:25] offset:176
-; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[24:25] offset:192
; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v0, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
@@ -157,7 +157,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v72 offset:124
; ALIGNED-NEXT: flat_store_byte v[24:25], v72 offset:128
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(19)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(22)
; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:228
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:232
@@ -180,6 +180,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v98 offset:60
; ALIGNED-NEXT: flat_store_byte v[24:25], v98 offset:64
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(29)
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:320
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:324
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:328
@@ -201,6 +202,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:32
; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:33
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(36)
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:272
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:276
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:280
@@ -222,6 +224,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v52 offset:12
; ALIGNED-NEXT: flat_store_byte v[24:25], v52 offset:16
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(43)
; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:292
; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:296
@@ -243,7 +246,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[24:25], v80 offset:2
; ALIGNED-NEXT: flat_store_byte v[24:25], v80
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(40)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(50)
; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:112
; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:116
; ALIGNED-NEXT: buffer_store_dword v70, off, s[0:3], s32 offset:120
@@ -265,6 +268,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v68 offset:236
; ALIGNED-NEXT: flat_store_byte v[20:21], v68 offset:237
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(48)
; ALIGNED-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:128
; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:132
; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:136
@@ -881,6 +885,11 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: s_clause 0xf
; ALIGNED-NEXT: global_load_dwordx4 v[16:19], v[24:25], off offset:240
; ALIGNED-NEXT: global_load_dwordx4 v[20:23], v[24:25], off offset:224
+; ALIGNED-NEXT: global_load_dwordx4 v[100:103], v[24:25], off offset:208
+; ALIGNED-NEXT: global_load_dwordx4 v[96:99], v[24:25], off offset:192
+; ALIGNED-NEXT: global_load_dwordx4 v[81:84], v[24:25], off offset:176
+; ALIGNED-NEXT: global_load_dwordx4 v[66:69], v[24:25], off offset:160
+; ALIGNED-NEXT: global_load_dwordx4 v[52:55], v[24:25], off offset:144
; ALIGNED-NEXT: global_load_dwordx4 v[4:7], v[24:25], off
; ALIGNED-NEXT: global_load_dwordx4 v[8:11], v[24:25], off offset:16
; ALIGNED-NEXT: global_load_dwordx4 v[12:15], v[24:25], off offset:32
@@ -890,11 +899,6 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: global_load_dwordx4 v[26:29], v[24:25], off offset:96
; ALIGNED-NEXT: global_load_dwordx4 v[32:35], v[24:25], off offset:112
; ALIGNED-NEXT: global_load_dwordx4 v[44:47], v[24:25], off offset:128
-; ALIGNED-NEXT: global_load_dwordx4 v[52:55], v[24:25], off offset:144
-; ALIGNED-NEXT: global_load_dwordx4 v[66:69], v[24:25], off offset:160
-; ALIGNED-NEXT: global_load_dwordx4 v[81:84], v[24:25], off offset:176
-; ALIGNED-NEXT: global_load_dwordx4 v[96:99], v[24:25], off offset:192
-; ALIGNED-NEXT: global_load_dwordx4 v[100:103], v[24:25], off offset:208
; ALIGNED-NEXT: s_waitcnt vmcnt(15)
; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64
; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:68
@@ -2468,6 +2472,7 @@ define void @memcpy_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x34
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:203
; ALIGNED-NEXT: buffer_load_ubyte v116, v1, s[0:3], 0 offen offset:255
; ALIGNED-NEXT: buffer_load_ubyte v117, v1, s[0:3], 0 offen offset:254
; ALIGNED-NEXT: buffer_load_ubyte v118, v1, s[0:3], 0 offen offset:253
@@ -2520,10 +2525,10 @@ define void @memcpy_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:206
; ALIGNED-NEXT: buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:205
; ALIGNED-NEXT: buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:204
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(52)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:129
; ALIGNED-NEXT: buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:202
; ALIGNED-NEXT: buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:201
; ALIGNED-NEXT: buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:200
@@ -2586,8 +2591,8 @@ define void @memcpy_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:143
; ALIGNED-NEXT: buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:142
; ALIGNED-NEXT: buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:141
-; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:140
; ALIGNED-NEXT: s_clause 0xa
+; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:140
; ALIGNED-NEXT: buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:139
; ALIGNED-NEXT: buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:138
; ALIGNED-NEXT: buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:137
@@ -2598,8 +2603,7 @@ define void @memcpy_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v113, v1, s[0:3], 0 offen offset:132
; ALIGNED-NEXT: buffer_load_ubyte v114, v1, s[0:3], 0 offen offset:131
; ALIGNED-NEXT: buffer_load_ubyte v115, v1, s[0:3], 0 offen offset:130
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:129
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:128
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -3616,27 +3620,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; CHECK-NEXT: .LBB4_1: ; %static-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3e
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:44
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:48
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:52
-; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:56
-; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:60
-; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:76
-; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
-; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
-; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
-; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
-; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:72
-; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:68
-; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:236
; CHECK-NEXT: buffer_load_dword v26, v2, s[0:3], 0 offen offset:252
; CHECK-NEXT: buffer_load_dword v25, v2, s[0:3], 0 offen offset:248
; CHECK-NEXT: buffer_load_dword v24, v2, s[0:3], 0 offen offset:244
; CHECK-NEXT: buffer_load_dword v23, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:236
; CHECK-NEXT: buffer_load_dword v21, v2, s[0:3], 0 offen offset:232
; CHECK-NEXT: buffer_load_dword v20, v2, s[0:3], 0 offen offset:228
; CHECK-NEXT: buffer_load_dword v19, v2, s[0:3], 0 offen offset:224
@@ -3648,14 +3636,6 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; CHECK-NEXT: buffer_load_dword v33, v2, s[0:3], 0 offen offset:200
; CHECK-NEXT: buffer_load_dword v32, v2, s[0:3], 0 offen offset:196
; CHECK-NEXT: buffer_load_dword v31, v2, s[0:3], 0 offen offset:192
-; CHECK-NEXT: buffer_load_dword v38, v2, s[0:3], 0 offen offset:108
-; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:124
-; CHECK-NEXT: buffer_load_dword v50, v2, s[0:3], 0 offen offset:120
-; CHECK-NEXT: buffer_load_dword v49, v2, s[0:3], 0 offen offset:116
-; CHECK-NEXT: buffer_load_dword v48, v2, s[0:3], 0 offen offset:112
-; CHECK-NEXT: buffer_load_dword v37, v2, s[0:3], 0 offen offset:104
-; CHECK-NEXT: buffer_load_dword v36, v2, s[0:3], 0 offen offset:100
-; CHECK-NEXT: buffer_load_dword v35, v2, s[0:3], 0 offen offset:96
; CHECK-NEXT: buffer_load_dword v55, v2, s[0:3], 0 offen offset:188
; CHECK-NEXT: buffer_load_dword v54, v2, s[0:3], 0 offen offset:184
; CHECK-NEXT: buffer_load_dword v53, v2, s[0:3], 0 offen offset:180
@@ -3672,13 +3652,37 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; CHECK-NEXT: buffer_load_dword v82, v2, s[0:3], 0 offen offset:136
; CHECK-NEXT: buffer_load_dword v81, v2, s[0:3], 0 offen offset:132
; CHECK-NEXT: buffer_load_dword v80, v2, s[0:3], 0 offen offset:128
-; CHECK-NEXT: buffer_load_dword v84, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v85, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v86, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:124
+; CHECK-NEXT: buffer_load_dword v50, v2, s[0:3], 0 offen offset:120
+; CHECK-NEXT: buffer_load_dword v49, v2, s[0:3], 0 offen offset:116
+; CHECK-NEXT: buffer_load_dword v48, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT: buffer_load_dword v38, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT: buffer_load_dword v37, v2, s[0:3], 0 offen offset:104
+; CHECK-NEXT: buffer_load_dword v36, v2, s[0:3], 0 offen offset:100
+; CHECK-NEXT: buffer_load_dword v35, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:80
+; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:48
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:52
+; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:56
+; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:60
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:44
; CHECK-NEXT: buffer_load_dword v96, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v97, v2, s[0:3], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v98, v2, s[0:3], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v99, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v84, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v85, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v86, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v87, v2, s[0:3], 0 offen offset:12
; CHECK-NEXT: v_add_co_u32 v100, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 0x100
@@ -3688,29 +3692,35 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
; CHECK-NEXT: v_add_co_ci_u32_e64 v103, null, 0, v101, vcc_lo
-; CHECK-NEXT: s_waitcnt vmcnt(43)
+; CHECK-NEXT: s_waitcnt vmcnt(60)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[23:26] offset:192
-; CHECK-NEXT: s_waitcnt vmcnt(40)
+; CHECK-NEXT: s_waitcnt vmcnt(56)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[19:22] offset:176
-; CHECK-NEXT: s_waitcnt vmcnt(36)
+; CHECK-NEXT: s_waitcnt vmcnt(52)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[27:30] offset:160
-; CHECK-NEXT: s_waitcnt vmcnt(32)
+; CHECK-NEXT: s_waitcnt vmcnt(48)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[31:34] offset:144
-; CHECK-NEXT: s_waitcnt vmcnt(20)
+; CHECK-NEXT: s_waitcnt vmcnt(44)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[52:55] offset:128
-; CHECK-NEXT: s_waitcnt vmcnt(16)
+; CHECK-NEXT: s_waitcnt vmcnt(40)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[64:67] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(12)
+; CHECK-NEXT: s_waitcnt vmcnt(36)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[68:71] offset:96
-; CHECK-NEXT: s_waitcnt vmcnt(8)
+; CHECK-NEXT: s_waitcnt vmcnt(32)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[80:83] offset:128
+; CHECK-NEXT: s_waitcnt vmcnt(28)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[48:51] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(24)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[35:38] offset:48
+; CHECK-NEXT: s_waitcnt vmcnt(20)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[15:18] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(16)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[11:14] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(12)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[7:10]
+; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[3:6] offset:32
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[96:99] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[84:87]
@@ -4226,11 +4236,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:165
; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:168
; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:169
; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:170
; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:173
; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:174
; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:175
@@ -4242,11 +4252,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:181
; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:182
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:184
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:185
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:186
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:189
; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:190
; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:191
@@ -4258,19 +4268,19 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:197
; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:198
; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:204
; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:200
; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:201
; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:202
; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:204
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:205
; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:206
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:212
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:208
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:209
; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:210
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:212
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:213
; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:214
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:215
@@ -4291,11 +4301,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:229
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:230
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:236
; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:232
; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:233
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:234
; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:235
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:236
; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:237
; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:238
; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:239
@@ -4440,11 +4450,11 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:245
; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:246
; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:252
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:248
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:249
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:250
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:252
; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v64
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v108, 8, v109
@@ -4516,9 +4526,9 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v15, 8, v20
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v14, 8, v16
-; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: v_lshl_or_b32 v56, v10, 8, v11
; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 16, v4
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v10, 8, v11
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v7, 8, v9
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -5335,14 +5345,14 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; UNROLL3-NEXT: .LBB4_1: ; %static-memcpy-expansion-main-body
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0xb
-; UNROLL3-NEXT: buffer_load_dword v4, v3, s[0:3], 0 offen
-; UNROLL3-NEXT: buffer_load_dword v5, v3, s[0:3], 0 offen offset:4
-; UNROLL3-NEXT: buffer_load_dword v6, v3, s[0:3], 0 offen offset:8
-; UNROLL3-NEXT: buffer_load_dword v7, v3, s[0:3], 0 offen offset:12
; UNROLL3-NEXT: buffer_load_dword v8, v3, s[0:3], 0 offen offset:16
; UNROLL3-NEXT: buffer_load_dword v9, v3, s[0:3], 0 offen offset:20
; UNROLL3-NEXT: buffer_load_dword v10, v3, s[0:3], 0 offen offset:24
; UNROLL3-NEXT: buffer_load_dword v11, v3, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT: buffer_load_dword v4, v3, s[0:3], 0 offen
+; UNROLL3-NEXT: buffer_load_dword v5, v3, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT: buffer_load_dword v6, v3, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT: buffer_load_dword v7, v3, s[0:3], 0 offen offset:12
; UNROLL3-NEXT: buffer_load_dword v12, v3, s[0:3], 0 offen offset:32
; UNROLL3-NEXT: buffer_load_dword v13, v3, s[0:3], 0 offen offset:36
; UNROLL3-NEXT: buffer_load_dword v14, v3, s[0:3], 0 offen offset:40
@@ -5353,8 +5363,9 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: v_add_nc_u32_e32 v3, 48, v3
; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
-; UNROLL3-NEXT: s_waitcnt vmcnt(4)
+; UNROLL3-NEXT: s_waitcnt vmcnt(8)
; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[8:11] offset:16
+; UNROLL3-NEXT: s_waitcnt vmcnt(4)
; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[4:7]
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
; UNROLL3-NEXT: flat_store_dwordx4 v[16:17], v[12:15] offset:32
@@ -5564,8 +5575,9 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v3, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
; ALIGNED-NEXT: flat_load_dwordx4 v[24:27], v[2:3] offset:64
-; ALIGNED-NEXT: flat_load_dwordx4 v[28:31], v[2:3] offset:16
; ALIGNED-NEXT: flat_load_dwordx4 v[32:35], v[2:3] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[28:31], v[2:3] offset:16
+; ALIGNED-NEXT: flat_load_dwordx4 v[100:103], v[2:3]
; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[36:37]
; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
@@ -5577,7 +5589,6 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: flat_load_dwordx4 v[52:55], v[36:37] offset:160
; ALIGNED-NEXT: flat_load_dwordx4 v[48:51], v[36:37] offset:176
; ALIGNED-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:192
-; ALIGNED-NEXT: flat_load_dwordx4 v[100:103], v[2:3]
; ALIGNED-NEXT: s_add_u32 s4, s4, 0xffffff00
; ALIGNED-NEXT: s_addc_u32 s5, s5, -1
; ALIGNED-NEXT: s_waitcnt vmcnt(15) lgkmcnt(15)
@@ -5631,7 +5642,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v96 offset:60
; ALIGNED-NEXT: flat_store_byte v[0:1], v96 offset:64
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(28)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(29)
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:292
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:296
@@ -5653,6 +5664,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v24 offset:32
; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:33
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(36)
; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:248
@@ -5674,7 +5686,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v28 offset:12
; ALIGNED-NEXT: flat_store_byte v[0:1], v28 offset:16
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(32)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(43)
; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:256
; ALIGNED-NEXT: buffer_store_dword v101, off, s[0:3], s32 offset:260
; ALIGNED-NEXT: buffer_store_dword v102, off, s[0:3], s32 offset:264
@@ -5704,6 +5716,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[0:1], v32 offset:2
; ALIGNED-NEXT: flat_store_byte v[0:1], v32
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(40)
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:80
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:84
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:88
@@ -6203,8 +6216,10 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v38, null, 0, v19, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
; ALIGNED-NEXT: flat_load_dwordx4 v[24:27], v[18:19] offset:64
-; ALIGNED-NEXT: flat_load_dwordx4 v[33:36], v[18:19] offset:16
; ALIGNED-NEXT: flat_load_dwordx4 v[28:31], v[18:19] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[33:36], v[18:19] offset:16
+; ALIGNED-NEXT: flat_load_dwordx4 v[80:83], v[18:19]
+; ALIGNED-NEXT: flat_load_dwordx4 v[66:69], v[37:38] offset:192
; ALIGNED-NEXT: flat_load_dwordx4 v[0:3], v[37:38]
; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[37:38] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[37:38] offset:48
@@ -6215,8 +6230,6 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: flat_load_dwordx4 v[39:42], v[37:38] offset:144
; ALIGNED-NEXT: flat_load_dwordx4 v[43:46], v[37:38] offset:160
; ALIGNED-NEXT: flat_load_dwordx4 v[52:55], v[37:38] offset:176
-; ALIGNED-NEXT: flat_load_dwordx4 v[66:69], v[37:38] offset:192
-; ALIGNED-NEXT: flat_load_dwordx4 v[80:83], v[18:19]
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
; ALIGNED-NEXT: s_waitcnt vmcnt(15) lgkmcnt(15)
@@ -6270,7 +6283,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v96 offset:60
; ALIGNED-NEXT: flat_store_byte v[16:17], v96 offset:64
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(28)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(29)
; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:544
; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:548
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:552
@@ -6292,6 +6305,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v30 offset:32
; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:33
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(36)
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:504
@@ -6313,7 +6327,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v50 offset:12
; ALIGNED-NEXT: flat_store_byte v[16:17], v50 offset:16
-; ALIGNED-NEXT: s_waitcnt lgkmcnt(32)
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(43)
; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:512
; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:516
; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:520
@@ -6335,6 +6349,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[16:17], v70 offset:2
; ALIGNED-NEXT: flat_store_byte v[16:17], v70
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(50)
; ALIGNED-NEXT: buffer_store_dword v66, off, s[0:3], s32 offset:336
; ALIGNED-NEXT: buffer_store_dword v67, off, s[0:3], s32 offset:340
; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:344
@@ -6356,6 +6371,7 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v66 offset:236
; ALIGNED-NEXT: flat_store_byte v[20:21], v66 offset:237
+; ALIGNED-NEXT: s_waitcnt lgkmcnt(48)
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:352
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:356
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:360
@@ -7097,6 +7113,11 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: s_clause 0xf
; ALIGNED-NEXT: global_load_dwordx4 v[16:19], v[2:3], off offset:240
; ALIGNED-NEXT: global_load_dwordx4 v[20:23], v[2:3], off offset:224
+; ALIGNED-NEXT: global_load_dwordx4 v[24:27], v[2:3], off offset:208
+; ALIGNED-NEXT: global_load_dwordx4 v[28:31], v[2:3], off offset:192
+; ALIGNED-NEXT: global_load_dwordx4 v[32:35], v[2:3], off offset:176
+; ALIGNED-NEXT: global_load_dwordx4 v[36:39], v[2:3], off offset:160
+; ALIGNED-NEXT: global_load_dwordx4 v[48:51], v[2:3], off offset:144
; ALIGNED-NEXT: global_load_dwordx4 v[4:7], v[2:3], off
; ALIGNED-NEXT: global_load_dwordx4 v[8:11], v[2:3], off offset:16
; ALIGNED-NEXT: global_load_dwordx4 v[12:15], v[2:3], off offset:32
@@ -7106,11 +7127,6 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_load_dwordx4 v[68:71], v[2:3], off offset:96
; ALIGNED-NEXT: global_load_dwordx4 v[64:67], v[2:3], off offset:112
; ALIGNED-NEXT: global_load_dwordx4 v[52:55], v[2:3], off offset:128
-; ALIGNED-NEXT: global_load_dwordx4 v[48:51], v[2:3], off offset:144
-; ALIGNED-NEXT: global_load_dwordx4 v[36:39], v[2:3], off offset:160
-; ALIGNED-NEXT: global_load_dwordx4 v[32:35], v[2:3], off offset:176
-; ALIGNED-NEXT: global_load_dwordx4 v[28:31], v[2:3], off offset:192
-; ALIGNED-NEXT: global_load_dwordx4 v[24:27], v[2:3], off offset:208
; ALIGNED-NEXT: v_add_co_u32 v2, vcc_lo, 0x100, v2
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0xffffff00
@@ -7727,6 +7743,11 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: s_clause 0xf
; ALIGNED-NEXT: global_load_dwordx4 v[16:19], v[14:15], off offset:240
; ALIGNED-NEXT: global_load_dwordx4 v[20:23], v[14:15], off offset:224
+; ALIGNED-NEXT: global_load_dwordx4 v[64:67], v[14:15], off offset:208
+; ALIGNED-NEXT: global_load_dwordx4 v[82:85], v[14:15], off offset:192
+; ALIGNED-NEXT: global_load_dwordx4 v[68:71], v[14:15], off offset:176
+; ALIGNED-NEXT: global_load_dwordx4 v[54:57], v[14:15], off offset:160
+; ALIGNED-NEXT: global_load_dwordx4 v[44:47], v[14:15], off offset:144
; ALIGNED-NEXT: global_load_dwordx4 v[0:3], v[14:15], off
; ALIGNED-NEXT: global_load_dwordx4 v[4:7], v[14:15], off offset:16
; ALIGNED-NEXT: global_load_dwordx4 v[8:11], v[14:15], off offset:32
@@ -7736,11 +7757,6 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_load_dwordx4 v[40:43], v[14:15], off offset:96
; ALIGNED-NEXT: global_load_dwordx4 v[28:31], v[14:15], off offset:112
; ALIGNED-NEXT: global_load_dwordx4 v[34:37], v[14:15], off offset:128
-; ALIGNED-NEXT: global_load_dwordx4 v[44:47], v[14:15], off offset:144
-; ALIGNED-NEXT: global_load_dwordx4 v[54:57], v[14:15], off offset:160
-; ALIGNED-NEXT: global_load_dwordx4 v[68:71], v[14:15], off offset:176
-; ALIGNED-NEXT: global_load_dwordx4 v[82:85], v[14:15], off offset:192
-; ALIGNED-NEXT: global_load_dwordx4 v[64:67], v[14:15], off offset:208
; ALIGNED-NEXT: v_add_co_u32 v14, vcc_lo, 0xffffff00, v14
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v15, null, -1, v15, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
@@ -10211,14 +10227,15 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x4
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:251
; ALIGNED-NEXT: buffer_load_ubyte v124, v1, s[0:3], 0 offen offset:255
; ALIGNED-NEXT: buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:254
; ALIGNED-NEXT: buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:253
; ALIGNED-NEXT: buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:129
; ALIGNED-NEXT: buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:250
; ALIGNED-NEXT: buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:249
; ALIGNED-NEXT: buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:248
@@ -10281,8 +10298,8 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:191
; ALIGNED-NEXT: buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:190
; ALIGNED-NEXT: buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: s_clause 0x3a
+; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:187
; ALIGNED-NEXT: buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:186
; ALIGNED-NEXT: buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:185
@@ -10341,8 +10358,7 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v121, v1, s[0:3], 0 offen offset:132
; ALIGNED-NEXT: buffer_load_ubyte v122, v1, s[0:3], 0 offen offset:131
; ALIGNED-NEXT: buffer_load_ubyte v123, v1, s[0:3], 0 offen offset:130
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:129
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:128
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -11264,6 +11280,7 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x34
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1995
; ALIGNED-NEXT: buffer_load_ubyte v116, v1, s[0:3], 0 offen offset:2047
; ALIGNED-NEXT: buffer_load_ubyte v117, v1, s[0:3], 0 offen offset:2046
; ALIGNED-NEXT: buffer_load_ubyte v118, v1, s[0:3], 0 offen offset:2045
@@ -11316,10 +11333,10 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v125, v1, s[0:3], 0 offen offset:1998
; ALIGNED-NEXT: buffer_load_ubyte v126, v1, s[0:3], 0 offen offset:1997
; ALIGNED-NEXT: buffer_load_ubyte v127, v1, s[0:3], 0 offen offset:1996
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1995
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(52)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:712 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
+; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1921
; ALIGNED-NEXT: buffer_load_ubyte v3, v1, s[0:3], 0 offen offset:1994
; ALIGNED-NEXT: buffer_load_ubyte v4, v1, s[0:3], 0 offen offset:1993
; ALIGNED-NEXT: buffer_load_ubyte v5, v1, s[0:3], 0 offen offset:1992
@@ -11382,8 +11399,8 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v86, v1, s[0:3], 0 offen offset:1935
; ALIGNED-NEXT: buffer_load_ubyte v87, v1, s[0:3], 0 offen offset:1934
; ALIGNED-NEXT: buffer_load_ubyte v96, v1, s[0:3], 0 offen offset:1933
-; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:1932
; ALIGNED-NEXT: s_clause 0xa
+; ALIGNED-NEXT: buffer_load_ubyte v97, v1, s[0:3], 0 offen offset:1932
; ALIGNED-NEXT: buffer_load_ubyte v98, v1, s[0:3], 0 offen offset:1931
; ALIGNED-NEXT: buffer_load_ubyte v99, v1, s[0:3], 0 offen offset:1930
; ALIGNED-NEXT: buffer_load_ubyte v100, v1, s[0:3], 0 offen offset:1929
@@ -11394,8 +11411,7 @@ define void @memmove_p5_p5_sz2048(ptr addrspace(5) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v113, v1, s[0:3], 0 offen offset:1924
; ALIGNED-NEXT: buffer_load_ubyte v114, v1, s[0:3], 0 offen offset:1923
; ALIGNED-NEXT: buffer_load_ubyte v115, v1, s[0:3], 0 offen offset:1922
-; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1921
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen offset:1920
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -12505,66 +12521,66 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: .LBB9_2: ; %memmove_fwd_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3e
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
-; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
-; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
-; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:44
-; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:76
-; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:92
-; CHECK-NEXT: buffer_load_dword v21, v2, s[0:3], 0 offen offset:88
-; CHECK-NEXT: buffer_load_dword v20, v2, s[0:3], 0 offen offset:84
-; CHECK-NEXT: buffer_load_dword v19, v2, s[0:3], 0 offen offset:80
-; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:72
-; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:68
-; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v26, v2, s[0:3], 0 offen offset:172
-; CHECK-NEXT: buffer_load_dword v30, v2, s[0:3], 0 offen offset:188
-; CHECK-NEXT: buffer_load_dword v29, v2, s[0:3], 0 offen offset:184
-; CHECK-NEXT: buffer_load_dword v28, v2, s[0:3], 0 offen offset:180
-; CHECK-NEXT: buffer_load_dword v27, v2, s[0:3], 0 offen offset:176
-; CHECK-NEXT: buffer_load_dword v25, v2, s[0:3], 0 offen offset:168
-; CHECK-NEXT: buffer_load_dword v24, v2, s[0:3], 0 offen offset:164
-; CHECK-NEXT: buffer_load_dword v23, v2, s[0:3], 0 offen offset:160
-; CHECK-NEXT: buffer_load_dword v34, v2, s[0:3], 0 offen offset:204
-; CHECK-NEXT: buffer_load_dword v38, v2, s[0:3], 0 offen offset:220
-; CHECK-NEXT: buffer_load_dword v37, v2, s[0:3], 0 offen offset:216
-; CHECK-NEXT: buffer_load_dword v36, v2, s[0:3], 0 offen offset:212
-; CHECK-NEXT: buffer_load_dword v35, v2, s[0:3], 0 offen offset:208
-; CHECK-NEXT: buffer_load_dword v33, v2, s[0:3], 0 offen offset:200
-; CHECK-NEXT: buffer_load_dword v32, v2, s[0:3], 0 offen offset:196
-; CHECK-NEXT: buffer_load_dword v31, v2, s[0:3], 0 offen offset:192
-; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:236
; CHECK-NEXT: buffer_load_dword v55, v2, s[0:3], 0 offen offset:252
; CHECK-NEXT: buffer_load_dword v54, v2, s[0:3], 0 offen offset:248
; CHECK-NEXT: buffer_load_dword v53, v2, s[0:3], 0 offen offset:244
; CHECK-NEXT: buffer_load_dword v52, v2, s[0:3], 0 offen offset:240
+; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:236
; CHECK-NEXT: buffer_load_dword v50, v2, s[0:3], 0 offen offset:232
; CHECK-NEXT: buffer_load_dword v49, v2, s[0:3], 0 offen offset:228
; CHECK-NEXT: buffer_load_dword v48, v2, s[0:3], 0 offen offset:224
-; CHECK-NEXT: buffer_load_dword v67, v2, s[0:3], 0 offen offset:140
+; CHECK-NEXT: buffer_load_dword v38, v2, s[0:3], 0 offen offset:220
+; CHECK-NEXT: buffer_load_dword v37, v2, s[0:3], 0 offen offset:216
+; CHECK-NEXT: buffer_load_dword v36, v2, s[0:3], 0 offen offset:212
+; CHECK-NEXT: buffer_load_dword v35, v2, s[0:3], 0 offen offset:208
+; CHECK-NEXT: buffer_load_dword v34, v2, s[0:3], 0 offen offset:204
+; CHECK-NEXT: buffer_load_dword v33, v2, s[0:3], 0 offen offset:200
+; CHECK-NEXT: buffer_load_dword v32, v2, s[0:3], 0 offen offset:196
+; CHECK-NEXT: buffer_load_dword v31, v2, s[0:3], 0 offen offset:192
+; CHECK-NEXT: buffer_load_dword v30, v2, s[0:3], 0 offen offset:188
+; CHECK-NEXT: buffer_load_dword v29, v2, s[0:3], 0 offen offset:184
+; CHECK-NEXT: buffer_load_dword v28, v2, s[0:3], 0 offen offset:180
+; CHECK-NEXT: buffer_load_dword v27, v2, s[0:3], 0 offen offset:176
+; CHECK-NEXT: buffer_load_dword v26, v2, s[0:3], 0 offen offset:172
+; CHECK-NEXT: buffer_load_dword v25, v2, s[0:3], 0 offen offset:168
+; CHECK-NEXT: buffer_load_dword v24, v2, s[0:3], 0 offen offset:164
+; CHECK-NEXT: buffer_load_dword v23, v2, s[0:3], 0 offen offset:160
; CHECK-NEXT: buffer_load_dword v71, v2, s[0:3], 0 offen offset:156
; CHECK-NEXT: buffer_load_dword v70, v2, s[0:3], 0 offen offset:152
; CHECK-NEXT: buffer_load_dword v69, v2, s[0:3], 0 offen offset:148
; CHECK-NEXT: buffer_load_dword v68, v2, s[0:3], 0 offen offset:144
+; CHECK-NEXT: buffer_load_dword v67, v2, s[0:3], 0 offen offset:140
; CHECK-NEXT: buffer_load_dword v66, v2, s[0:3], 0 offen offset:136
; CHECK-NEXT: buffer_load_dword v65, v2, s[0:3], 0 offen offset:132
; CHECK-NEXT: buffer_load_dword v64, v2, s[0:3], 0 offen offset:128
-; CHECK-NEXT: buffer_load_dword v83, v2, s[0:3], 0 offen offset:108
+; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:76
+; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:72
+; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:68
+; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:64
+; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:44
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
+; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
+; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
; CHECK-NEXT: buffer_load_dword v87, v2, s[0:3], 0 offen offset:124
; CHECK-NEXT: buffer_load_dword v86, v2, s[0:3], 0 offen offset:120
; CHECK-NEXT: buffer_load_dword v85, v2, s[0:3], 0 offen offset:116
; CHECK-NEXT: buffer_load_dword v84, v2, s[0:3], 0 offen offset:112
+; CHECK-NEXT: buffer_load_dword v83, v2, s[0:3], 0 offen offset:108
; CHECK-NEXT: buffer_load_dword v82, v2, s[0:3], 0 offen offset:104
; CHECK-NEXT: buffer_load_dword v81, v2, s[0:3], 0 offen offset:100
; CHECK-NEXT: buffer_load_dword v80, v2, s[0:3], 0 offen offset:96
+; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:92
+; CHECK-NEXT: buffer_load_dword v21, v2, s[0:3], 0 offen offset:88
+; CHECK-NEXT: buffer_load_dword v20, v2, s[0:3], 0 offen offset:84
+; CHECK-NEXT: buffer_load_dword v19, v2, s[0:3], 0 offen offset:80
; CHECK-NEXT: buffer_load_dword v96, v2, s[0:3], 0 offen offset:48
; CHECK-NEXT: buffer_load_dword v97, v2, s[0:3], 0 offen offset:52
; CHECK-NEXT: buffer_load_dword v98, v2, s[0:3], 0 offen offset:56
@@ -12574,29 +12590,38 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
; CHECK-NEXT: s_add_u32 s4, s4, 0xffffff00
; CHECK-NEXT: s_addc_u32 s5, s5, -1
-; CHECK-NEXT: s_waitcnt vmcnt(23)
+; CHECK-NEXT: s_waitcnt vmcnt(60)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[52:55] offset:192
-; CHECK-NEXT: s_waitcnt vmcnt(20)
+; CHECK-NEXT: s_waitcnt vmcnt(56)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[48:51] offset:176
+; CHECK-NEXT: s_waitcnt vmcnt(52)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[35:38] offset:160
+; CHECK-NEXT: s_waitcnt vmcnt(48)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[31:34] offset:144
+; CHECK-NEXT: s_waitcnt vmcnt(44)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[27:30] offset:128
+; CHECK-NEXT: s_waitcnt vmcnt(40)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[23:26] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(15)
+; CHECK-NEXT: s_waitcnt vmcnt(36)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[68:71] offset:96
-; CHECK-NEXT: s_waitcnt vmcnt(12)
+; CHECK-NEXT: s_waitcnt vmcnt(32)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[64:67] offset:128
+; CHECK-NEXT: s_waitcnt vmcnt(28)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[15:18] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(24)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[11:14] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(20)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
+; CHECK-NEXT: s_waitcnt vmcnt(16)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, 0x100, v0
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
-; CHECK-NEXT: s_waitcnt vmcnt(7)
+; CHECK-NEXT: s_waitcnt vmcnt(12)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[84:87] offset:64
-; CHECK-NEXT: s_waitcnt vmcnt(4)
+; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[80:83] offset:48
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[19:22] offset:32
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[96:99]
@@ -12612,47 +12637,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: .LBB9_5: ; %memmove_bwd_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_clause 0x3e
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen offset:1792
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:1796
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:1800
-; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:1804
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:1808
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:1812
-; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:1816
-; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:1820
-; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:1824
-; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:1828
-; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:1832
-; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:1836
-; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:1868
-; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:1884
-; CHECK-NEXT: buffer_load_dword v21, v2, s[0:3], 0 offen offset:1880
-; CHECK-NEXT: buffer_load_dword v20, v2, s[0:3], 0 offen offset:1876
-; CHECK-NEXT: buffer_load_dword v19, v2, s[0:3], 0 offen offset:1872
-; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:1864
-; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:1860
-; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:1856
-; CHECK-NEXT: buffer_load_dword v26, v2, s[0:3], 0 offen offset:2028
; CHECK-NEXT: buffer_load_dword v30, v2, s[0:3], 0 offen offset:2044
; CHECK-NEXT: buffer_load_dword v29, v2, s[0:3], 0 offen offset:2040
; CHECK-NEXT: buffer_load_dword v28, v2, s[0:3], 0 offen offset:2036
; CHECK-NEXT: buffer_load_dword v27, v2, s[0:3], 0 offen offset:2032
+; CHECK-NEXT: buffer_load_dword v26, v2, s[0:3], 0 offen offset:2028
; CHECK-NEXT: buffer_load_dword v25, v2, s[0:3], 0 offen offset:2024
; CHECK-NEXT: buffer_load_dword v24, v2, s[0:3], 0 offen offset:2020
; CHECK-NEXT: buffer_load_dword v23, v2, s[0:3], 0 offen offset:2016
-; CHECK-NEXT: buffer_load_dword v34, v2, s[0:3], 0 offen offset:1996
; CHECK-NEXT: buffer_load_dword v38, v2, s[0:3], 0 offen offset:2012
; CHECK-NEXT: buffer_load_dword v37, v2, s[0:3], 0 offen offset:2008
; CHECK-NEXT: buffer_load_dword v36, v2, s[0:3], 0 offen offset:2004
; CHECK-NEXT: buffer_load_dword v35, v2, s[0:3], 0 offen offset:2000
+; CHECK-NEXT: buffer_load_dword v34, v2, s[0:3], 0 offen offset:1996
; CHECK-NEXT: buffer_load_dword v33, v2, s[0:3], 0 offen offset:1992
; CHECK-NEXT: buffer_load_dword v32, v2, s[0:3], 0 offen offset:1988
; CHECK-NEXT: buffer_load_dword v31, v2, s[0:3], 0 offen offset:1984
-; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:1964
; CHECK-NEXT: buffer_load_dword v55, v2, s[0:3], 0 offen offset:1980
; CHECK-NEXT: buffer_load_dword v54, v2, s[0:3], 0 offen offset:1976
; CHECK-NEXT: buffer_load_dword v53, v2, s[0:3], 0 offen offset:1972
; CHECK-NEXT: buffer_load_dword v52, v2, s[0:3], 0 offen offset:1968
+; CHECK-NEXT: buffer_load_dword v51, v2, s[0:3], 0 offen offset:1964
; CHECK-NEXT: buffer_load_dword v50, v2, s[0:3], 0 offen offset:1960
; CHECK-NEXT: buffer_load_dword v49, v2, s[0:3], 0 offen offset:1956
; CHECK-NEXT: buffer_load_dword v48, v2, s[0:3], 0 offen offset:1952
@@ -12664,6 +12669,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: buffer_load_dword v70, v2, s[0:3], 0 offen offset:1928
; CHECK-NEXT: buffer_load_dword v69, v2, s[0:3], 0 offen offset:1924
; CHECK-NEXT: buffer_load_dword v68, v2, s[0:3], 0 offen offset:1920
+; CHECK-NEXT: buffer_load_dword v18, v2, s[0:3], 0 offen offset:1868
+; CHECK-NEXT: buffer_load_dword v17, v2, s[0:3], 0 offen offset:1864
+; CHECK-NEXT: buffer_load_dword v16, v2, s[0:3], 0 offen offset:1860
+; CHECK-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:1856
+; CHECK-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:1824
+; CHECK-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:1828
+; CHECK-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:1832
+; CHECK-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:1836
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:1808
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:1812
+; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:1816
+; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:1820
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen offset:1792
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:1796
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:1800
+; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:1804
; CHECK-NEXT: buffer_load_dword v83, v2, s[0:3], 0 offen offset:1916
; CHECK-NEXT: buffer_load_dword v82, v2, s[0:3], 0 offen offset:1912
; CHECK-NEXT: buffer_load_dword v81, v2, s[0:3], 0 offen offset:1908
@@ -12672,6 +12693,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: buffer_load_dword v86, v2, s[0:3], 0 offen offset:1896
; CHECK-NEXT: buffer_load_dword v85, v2, s[0:3], 0 offen offset:1892
; CHECK-NEXT: buffer_load_dword v84, v2, s[0:3], 0 offen offset:1888
+; CHECK-NEXT: buffer_load_dword v22, v2, s[0:3], 0 offen offset:1884
+; CHECK-NEXT: buffer_load_dword v21, v2, s[0:3], 0 offen offset:1880
+; CHECK-NEXT: buffer_load_dword v20, v2, s[0:3], 0 offen offset:1876
+; CHECK-NEXT: buffer_load_dword v19, v2, s[0:3], 0 offen offset:1872
; CHECK-NEXT: buffer_load_dword v96, v2, s[0:3], 0 offen offset:1840
; CHECK-NEXT: buffer_load_dword v97, v2, s[0:3], 0 offen offset:1844
; CHECK-NEXT: buffer_load_dword v98, v2, s[0:3], 0 offen offset:1848
@@ -12681,33 +12706,38 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; CHECK-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v2
; CHECK-NEXT: s_add_u32 s4, s4, 0x100
; CHECK-NEXT: s_addc_u32 s5, s5, 0
-; CHECK-NEXT: s_waitcnt vmcnt(39)
+; CHECK-NEXT: s_waitcnt vmcnt(60)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[27:30] offset:192
-; CHECK-NEXT: s_waitcnt vmcnt(36)
+; CHECK-NEXT: s_waitcnt vmcnt(56)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[23:26] offset:176
-; CHECK-NEXT: s_waitcnt vmcnt(31)
+; CHECK-NEXT: s_waitcnt vmcnt(52)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[35:38] offset:160
-; CHECK-NEXT: s_waitcnt vmcnt(28)
+; CHECK-NEXT: s_waitcnt vmcnt(48)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[31:34] offset:144
-; CHECK-NEXT: s_waitcnt vmcnt(23)
+; CHECK-NEXT: s_waitcnt vmcnt(44)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[52:55] offset:128
-; CHECK-NEXT: s_waitcnt vmcnt(20)
+; CHECK-NEXT: s_waitcnt vmcnt(40)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[48:51] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(16)
+; CHECK-NEXT: s_waitcnt vmcnt(36)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[64:67] offset:96
-; CHECK-NEXT: s_waitcnt vmcnt(12)
+; CHECK-NEXT: s_waitcnt vmcnt(32)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[68:71] offset:128
+; CHECK-NEXT: s_waitcnt vmcnt(28)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[15:18] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(24)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[11:14] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(20)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
+; CHECK-NEXT: s_waitcnt vmcnt(16)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff00, v0
; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
; CHECK-NEXT: s_cmp_eq_u64 s[4:5], 0
-; CHECK-NEXT: s_waitcnt vmcnt(8)
+; CHECK-NEXT: s_waitcnt vmcnt(12)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[80:83] offset:64
-; CHECK-NEXT: s_waitcnt vmcnt(4)
+; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[84:87] offset:48
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[19:22] offset:32
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[96:99]
@@ -13250,11 +13280,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:165
; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:168
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:169
; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:170
; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:173
; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:174
; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:175
@@ -13266,11 +13296,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:181
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:182
; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:184
; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:185
; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:186
; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:189
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:190
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:191
@@ -13282,27 +13312,27 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:197
; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:198
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:204
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:200
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:201
; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:202
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:204
; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:205
; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:206
; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:212
; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:208
; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:209
; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:210
; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:212
; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:229
; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:230
; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:236
; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:232
; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:233
; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:234
; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:235
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:236
; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:237
; ALIGNED-NEXT: s_clause 0x17
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:238
@@ -13315,11 +13345,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:245
; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:246
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:252
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:248
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:249
; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:250
; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:252
; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:253
; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:254
; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:255
@@ -14751,11 +14781,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:1957
; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:1958
; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:1959
+; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1964
; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:1960
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:1961
; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:1962
; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:1963
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1964
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:1965
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1966
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:1967
@@ -14767,11 +14797,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:1973
; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:1974
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:1975
+; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:1980
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:1976
; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:1977
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:1978
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:1979
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:1980
; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:1981
; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:1982
; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:1983
@@ -14783,19 +14813,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1989
; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1990
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:1991
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1996
; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:1992
; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1993
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1994
; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1995
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1996
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:1997
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1998
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1999
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:2004
; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:2000
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:2001
; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:2002
; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:2003
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:2004
; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:2005
; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:2006
; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:2007
@@ -14816,19 +14846,19 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:2037
; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:2038
; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:2039
+; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:2044
; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:2040
; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:2041
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:2042
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:2043
-; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:2044
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:2045
; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:2046
; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:2047
+; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1797
; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:1793
; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:1794
; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:1795
; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:1796
-; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1797
; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:1798
; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:1799
; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:1804
@@ -14882,7 +14912,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v87, v118, 8, v41
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1941
-; ALIGNED-NEXT: s_waitcnt vmcnt(21)
+; ALIGNED-NEXT: s_waitcnt vmcnt(25)
; ALIGNED-NEXT: buffer_store_dword v97, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1954
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
@@ -14984,11 +15014,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:2021
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:2022
; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:2023
+; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:2028
; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:2024
; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:2025
; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:2026
; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:2027
-; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:2028
; ALIGNED-NEXT: v_lshl_or_b32 v77, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v34
; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v32
@@ -15020,11 +15050,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v126, 8, v1
; ALIGNED-NEXT: v_lshl_or_b32 v15, v4, 16, v3
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v108
; ALIGNED-NEXT: v_lshl_or_b32 v4, v94, 8, v106
; ALIGNED-NEXT: v_lshl_or_b32 v8, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v124
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v110, 8, v120
; ALIGNED-NEXT: v_lshl_or_b32 v5, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v90, 8, v92
@@ -15856,14 +15888,14 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; UNROLL3-NEXT: .LBB9_2: ; %memmove_fwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0xb
-; UNROLL3-NEXT: buffer_load_dword v6, v5, s[0:3], 0 offen
-; UNROLL3-NEXT: buffer_load_dword v7, v5, s[0:3], 0 offen offset:4
-; UNROLL3-NEXT: buffer_load_dword v8, v5, s[0:3], 0 offen offset:8
-; UNROLL3-NEXT: buffer_load_dword v9, v5, s[0:3], 0 offen offset:12
; UNROLL3-NEXT: buffer_load_dword v10, v5, s[0:3], 0 offen offset:16
; UNROLL3-NEXT: buffer_load_dword v11, v5, s[0:3], 0 offen offset:20
; UNROLL3-NEXT: buffer_load_dword v12, v5, s[0:3], 0 offen offset:24
; UNROLL3-NEXT: buffer_load_dword v13, v5, s[0:3], 0 offen offset:28
+; UNROLL3-NEXT: buffer_load_dword v6, v5, s[0:3], 0 offen
+; UNROLL3-NEXT: buffer_load_dword v7, v5, s[0:3], 0 offen offset:4
+; UNROLL3-NEXT: buffer_load_dword v8, v5, s[0:3], 0 offen offset:8
+; UNROLL3-NEXT: buffer_load_dword v9, v5, s[0:3], 0 offen offset:12
; UNROLL3-NEXT: buffer_load_dword v14, v5, s[0:3], 0 offen offset:32
; UNROLL3-NEXT: buffer_load_dword v15, v5, s[0:3], 0 offen offset:36
; UNROLL3-NEXT: buffer_load_dword v16, v5, s[0:3], 0 offen offset:40
@@ -15871,8 +15903,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; UNROLL3-NEXT: v_add_nc_u32_e32 v5, 48, v5
; UNROLL3-NEXT: s_add_u32 s4, s4, 0xffffffd0
; UNROLL3-NEXT: s_addc_u32 s5, s5, -1
-; UNROLL3-NEXT: s_waitcnt vmcnt(4)
+; UNROLL3-NEXT: s_waitcnt vmcnt(8)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[10:13] offset:16
+; UNROLL3-NEXT: s_waitcnt vmcnt(4)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[6:9]
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[14:17] offset:32
@@ -15925,14 +15958,14 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; UNROLL3-NEXT: .LBB9_6: ; %memmove_bwd_loop
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
; UNROLL3-NEXT: s_clause 0xb
-; UNROLL3-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:1968
-; UNROLL3-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:1972
-; UNROLL3-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:1976
-; UNROLL3-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:1980
; UNROLL3-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:1984
; UNROLL3-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:1988
; UNROLL3-NEXT: buffer_load_dword v11, v2, s[0:3], 0 offen offset:1992
; UNROLL3-NEXT: buffer_load_dword v12, v2, s[0:3], 0 offen offset:1996
+; UNROLL3-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:1968
+; UNROLL3-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:1972
+; UNROLL3-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:1976
+; UNROLL3-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:1980
; UNROLL3-NEXT: buffer_load_dword v13, v2, s[0:3], 0 offen offset:2000
; UNROLL3-NEXT: buffer_load_dword v14, v2, s[0:3], 0 offen offset:2004
; UNROLL3-NEXT: buffer_load_dword v15, v2, s[0:3], 0 offen offset:2008
@@ -15940,8 +15973,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; UNROLL3-NEXT: v_subrev_nc_u32_e32 v2, 48, v2
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
-; UNROLL3-NEXT: s_waitcnt vmcnt(4)
+; UNROLL3-NEXT: s_waitcnt vmcnt(8)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[9:12] offset:16
+; UNROLL3-NEXT: s_waitcnt vmcnt(4)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[5:8]
; UNROLL3-NEXT: s_waitcnt vmcnt(0)
; UNROLL3-NEXT: flat_store_dwordx4 v[3:4], v[13:16] offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
index 254dd2cfc9598..47e84cc2f2d07 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
@@ -848,16 +848,17 @@ define void @memmove_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[7:8] offset:23
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[9:10] offset:16
@@ -873,15 +874,15 @@ define void @memmove_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
@@ -915,16 +916,17 @@ define void @memmove_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
+; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
+; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:23
-; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:27
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:20
-; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[7:8] offset:23
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[9:10] offset:16
@@ -940,15 +942,15 @@ define void @memmove_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_clause 0x7
-; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
-; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:20
; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:24
; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v5, v2, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v6, v2, s[0:3], 0 offen offset:12
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[7:10] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
index d416048942304..3276f86682453 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
@@ -12,23 +12,24 @@ define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
@@ -49,26 +50,29 @@ define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX10-NEXT: s_waitcnt vmcnt(7)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX10-NEXT: s_waitcnt vmcnt(5)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(4)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX10-NEXT: s_waitcnt vmcnt(3)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v3, v5, v4
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
@@ -91,23 +95,24 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
@@ -129,26 +134,29 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
+; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX10-NEXT: s_waitcnt vmcnt(7)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: s_waitcnt vmcnt(6)
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
; GFX10-NEXT: s_waitcnt vmcnt(5)
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(4)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; GFX10-NEXT: s_waitcnt vmcnt(3)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
; GFX10-NEXT: v_or_b32_e32 v4, v4, v6
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_or_b32_e32 v3, v5, v4
; GFX10-NEXT: v_or_b32_e32 v0, v2, v0
@@ -172,23 +180,24 @@ define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
@@ -211,22 +220,23 @@ define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX10-NEXT: s_clause 0x7
; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(6)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: s_waitcnt vmcnt(5)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: s_waitcnt vmcnt(4)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_or_b32_e32 v3, v4, v7
@@ -253,23 +263,24 @@ define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_or_b32_e32 v3, v4, v7
@@ -293,22 +304,23 @@ define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX10-NEXT: s_clause 0x7
; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(6)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX10-NEXT: s_waitcnt vmcnt(4)
+; GFX10-NEXT: s_waitcnt vmcnt(5)
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX10-NEXT: s_waitcnt vmcnt(3)
+; GFX10-NEXT: s_waitcnt vmcnt(4)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX10-NEXT: s_waitcnt vmcnt(2)
+; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_or_b32_e32 v3, v4, v7
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 610d80e9bba09..e56125c5089be 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -669,13 +669,13 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: v_readlane_b32 s4, v17, 9
; W64-O0-NEXT: v_readlane_b32 s5, v17, 10
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(3)
; W64-O0-NEXT: global_store_dword v[3:4], v5, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
@@ -1106,12 +1106,13 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s7, v13, 3
; W64-O0-NEXT: s_mov_b64 exec, s[6:7]
; W64-O0-NEXT: v_readlane_b32 s4, v13, 1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b32 s5, 0x3ff
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_waitcnt vmcnt(1)
; W64-O0-NEXT: v_and_b32_e64 v1, v1, s5
; W64-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v1, s4
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 s[4:5], exec
; W64-O0-NEXT: v_writelane_b32 v13, s4, 10
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 6911408544888..031c38b49494c 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -689,13 +689,13 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: v_readlane_b32 s4, v17, 9
; W64-O0-NEXT: v_readlane_b32 s5, v17, 10
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(3)
; W64-O0-NEXT: global_store_dword v[3:4], v5, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
@@ -1137,12 +1137,13 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s7, v13, 3
; W64-O0-NEXT: s_mov_b64 exec, s[6:7]
; W64-O0-NEXT: v_readlane_b32 s4, v13, 1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b32 s5, 0x3ff
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_waitcnt vmcnt(1)
; W64-O0-NEXT: v_and_b32_e64 v1, v1, s5
; W64-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v1, s4
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 s[4:5], exec
; W64-O0-NEXT: v_writelane_b32 v13, s4, 10
@@ -1159,14 +1160,17 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readlane_b32 s4, v13, 0
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(3)
; W64-O0-NEXT: v_mov_b32_e32 v6, v4
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_mov_b32_e32 v0, v3
+; W64-O0-NEXT: s_waitcnt vmcnt(1)
; W64-O0-NEXT: v_mov_b32_e32 v4, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_mov_b32_e32 v5, v1
; W64-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
; W64-O0-NEXT: v_mov_b32_e32 v1, v6
diff --git a/llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll b/llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll
index 919cbb0e00025..cee99163edf9f 100644
--- a/llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll
@@ -48,17 +48,17 @@ define <3 x i64> @v3_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 20, v0
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: flat_load_dword v4, v[2:3] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
; CHECK-NEXT: v_mov_b32_e32 v3, -1
; CHECK-NEXT: flat_load_dword v1, v[0:1]
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_ashrrev_i32_e32 v0, v8, v5
; CHECK-NEXT: v_ashrrev_i32_e32 v2, v10, v7
+; CHECK-NEXT: v_mov_b32_e32 v5, -1
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_ashrrev_i32_e32 v4, v4, v1
; CHECK-NEXT: v_mov_b32_e32 v1, -1
-; CHECK-NEXT: v_mov_b32_e32 v5, -1
; CHECK-NEXT: s_setpc_b64 s[30:31]
%val = load <3 x i64>, ptr %arg0.ptr, !range !4, !noundef !{}
%shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !5, !noundef !{}
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index c5d6481e78065..d402a73adb598 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -2272,14 +2272,14 @@ define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: s_clause 0xd
; GFX1251-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:96
; GFX1251-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:112
-; GFX1251-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:224
-; GFX1251-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1] offset:240
-; GFX1251-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:128
; GFX1251-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:144
-; GFX1251-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:160
+; GFX1251-SDAG-NEXT: global_load_b128 v[16:19], v56, s[0:1] offset:128
; GFX1251-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:176
-; GFX1251-SDAG-NEXT: global_load_b128 v[32:35], v56, s[0:1] offset:192
+; GFX1251-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:160
; GFX1251-SDAG-NEXT: global_load_b128 v[36:39], v56, s[0:1] offset:208
+; GFX1251-SDAG-NEXT: global_load_b128 v[32:35], v56, s[0:1] offset:192
+; GFX1251-SDAG-NEXT: global_load_b128 v[12:15], v56, s[0:1] offset:240
+; GFX1251-SDAG-NEXT: global_load_b128 v[8:11], v56, s[0:1] offset:224
; GFX1251-SDAG-NEXT: global_load_b128 v[40:43], v56, s[0:1] offset:64
; GFX1251-SDAG-NEXT: global_load_b128 v[44:47], v56, s[0:1] offset:80
; GFX1251-SDAG-NEXT: global_load_b128 v[48:51], v56, s[0:1] offset:32
@@ -2289,13 +2289,13 @@ define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0xc
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[4:7], v[4:7], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0xb
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[8:11], v[8:11], 0 neg_lo:[1,1] neg_hi:[1,1]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[20:23], v[20:23], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0xa
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[12:15], v[12:15], 0 neg_lo:[1,1] neg_hi:[1,1]
-; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x9
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[16:19], v[16:19], 0 neg_lo:[1,1] neg_hi:[1,1]
+; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x9
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[28:31], v[28:31], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x8
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[20:23], v[20:23], 0 neg_lo:[1,1] neg_hi:[1,1]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[24:27], v[24:27], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_clause 0x3
; GFX1251-SDAG-NEXT: global_store_b128 v56, v[0:3], s[0:1] offset:96
; GFX1251-SDAG-NEXT: global_store_b128 v56, v[4:7], s[0:1] offset:112
@@ -2309,10 +2309,10 @@ define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[48:51], v[48:51], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x2
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[52:55], v[52:55], 0 neg_lo:[1,1] neg_hi:[1,1]
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[28:31], v[28:31], 0 neg_lo:[1,1] neg_hi:[1,1]
-; GFX1251-SDAG-NEXT: v_pk_add_f64 v[24:27], v[24:27], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[36:39], v[36:39], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[32:35], v[32:35], 0 neg_lo:[1,1] neg_hi:[1,1]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[12:15], v[12:15], 0 neg_lo:[1,1] neg_hi:[1,1]
+; GFX1251-SDAG-NEXT: v_pk_add_f64 v[8:11], v[8:11], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x1
; GFX1251-SDAG-NEXT: v_pk_add_f64 v[0:3], v[0:3], 0 neg_lo:[1,1] neg_hi:[1,1]
; GFX1251-SDAG-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index a7aadc64d750d..25ed5a722c4b6 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3494,11 +3494,11 @@ define hidden void @extract_v6i16(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_clause 0x3
-; GFX10-NEXT: global_load_ushort v2, v[0:1], off offset:6
; GFX10-NEXT: global_load_ushort v3, v[0:1], off
; GFX10-NEXT: global_load_ushort v8, v[0:1], off offset:2
+; GFX10-NEXT: global_load_ushort v2, v[0:1], off offset:6
; GFX10-NEXT: global_load_ushort v9, v[0:1], off offset:4
-; GFX10-NEXT: s_waitcnt vmcnt(1)
+; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_lshl_or_b32 v0, v8, 16, v3
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v9
diff --git a/llvm/test/CodeGen/AMDGPU/post-ra-soft-clause-dbg-info.ll b/llvm/test/CodeGen/AMDGPU/post-ra-soft-clause-dbg-info.ll
index 0ef9fc8f60351..8a0199e05214f 100644
--- a/llvm/test/CodeGen/AMDGPU/post-ra-soft-clause-dbg-info.ll
+++ b/llvm/test/CodeGen/AMDGPU/post-ra-soft-clause-dbg-info.ll
@@ -32,6 +32,38 @@ define amdgpu_kernel void @dbg_clause(ptr addrspace(1) %out, ptr addrspace(1) %a
ret void
}
+; Confirm that bundles are still reordered by use order even with debug info.
+
+define amdgpu_kernel void @dbg_clause_reorder(ptr addrspace(1) %out, ptr addrspace(1) %aptr) !dbg !12 {
+; GCN-LABEL: dbg_clause_reorder:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: global_load_dword v1, v0, s[2:3] offset:32
+; GCN-NEXT: ;DEBUG_VALUE: foo:b <- $vgpr1
+; GCN-NEXT: global_load_dword v2, v0, s[2:3]
+; GCN-NEXT: ;DEBUG_VALUE: foo:a <- $vgpr2
+; GCN-NEXT: s_waitcnt vmcnt(1)
+; GCN-NEXT: v_add_f32_e32 v1, 2.0, v1
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_add_f32_e32 v1, v1, v2
+; GCN-NEXT: global_store_dword v0, v1, s[0:1]
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
+ %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
+ %gep1 = getelementptr float, ptr addrspace(1) %gep0, i32 8
+ %a = load float, ptr addrspace(1) %gep0, align 4
+ call void @llvm.dbg.value(metadata float %a, metadata !8, metadata !DIExpression()), !dbg !9
+ %b = load float, ptr addrspace(1) %gep1, align 4
+ call void @llvm.dbg.value(metadata float %b, metadata !10, metadata !DIExpression()), !dbg !11
+ %fadd.0 = fadd float %b, 2.0
+ %fadd.1 = fadd float %fadd.0, %a
+ store float %fadd.1, ptr addrspace(1) %out.gep, align 4
+ ret void
+}
+
declare i32 @llvm.amdgcn.workitem.id.x() #0
declare void @llvm.dbg.value(metadata, metadata, metadata) #1
@@ -53,3 +85,4 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
!9 = !DILocation(line: 1, column: 42, scope: !4)
!10 = !DILocalVariable(name: "b", arg: 2, scope: !4, file: !1, line: 2)
!11 = !DILocation(line: 2, column: 42, scope: !4)
+!12 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
index c22e48557b876..adb1c58918211 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
@@ -306,38 +306,38 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX906-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload
; GFX906-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:16 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:60 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:68 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:72 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:76 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:80 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:84 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:88 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:92 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:96 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:100 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:104 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:108 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:112 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:116 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:120 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:124 ; 4-byte Folded Reload
-; GFX906-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:128 ; 4-byte Folded Reload
; GFX906-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
; GFX906-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload
; GFX906-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:140 ; 4-byte Folded Reload
; GFX906-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:144 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:116 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:120 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:124 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:128 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:100 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:104 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:108 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:112 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:84 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:88 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:92 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:96 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:68 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:72 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:76 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:80 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:60 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload
+; GFX906-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload
; GFX906-NEXT: v_readlane_b32 s30, v41, 0
; GFX906-NEXT: v_readlane_b32 s31, v41, 1
; GFX906-NEXT: s_mov_b32 s32, s33
@@ -348,7 +348,7 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: v_add_co_u32_e32 v0, vcc, 0x70, v2
; GFX906-NEXT: s_waitcnt vmcnt(32)
; GFX906-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX906-NEXT: s_waitcnt vmcnt(0)
+; GFX906-NEXT: s_waitcnt vmcnt(28)
; GFX906-NEXT: flat_store_dwordx4 v[0:1], v[32:35]
; GFX906-NEXT: s_waitcnt vmcnt(0)
; GFX906-NEXT: v_add_co_u32_e32 v0, vcc, 0x60, v2
@@ -688,44 +688,44 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:20 ; 4-byte Folded Reload
; GFX908-NEXT: buffer_load_dword v3, off, s[0:3], s33 offset:24 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:60 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:68 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:72 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:76 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:80 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:84 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:88 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:92 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:96 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:100 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:104 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:108 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:112 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:116 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:120 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:124 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:128 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
-; GFX908-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload
; GFX908-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:140 ; 4-byte Folded Reload
; GFX908-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:144 ; 4-byte Folded Reload
; GFX908-NEXT: buffer_load_dword v34, off, s[0:3], s33 offset:148 ; 4-byte Folded Reload
; GFX908-NEXT: buffer_load_dword v35, off, s[0:3], s33 offset:152 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v28, off, s[0:3], s33 offset:124 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v29, off, s[0:3], s33 offset:128 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v30, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v31, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v24, off, s[0:3], s33 offset:108 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v25, off, s[0:3], s33 offset:112 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v26, off, s[0:3], s33 offset:116 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v27, off, s[0:3], s33 offset:120 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v20, off, s[0:3], s33 offset:92 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v21, off, s[0:3], s33 offset:96 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v22, off, s[0:3], s33 offset:100 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v23, off, s[0:3], s33 offset:104 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v16, off, s[0:3], s33 offset:76 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v17, off, s[0:3], s33 offset:80 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v18, off, s[0:3], s33 offset:84 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v19, off, s[0:3], s33 offset:88 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v12, off, s[0:3], s33 offset:60 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v13, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v14, off, s[0:3], s33 offset:68 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v15, off, s[0:3], s33 offset:72 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v8, off, s[0:3], s33 offset:44 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v9, off, s[0:3], s33 offset:48 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v10, off, s[0:3], s33 offset:52 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v11, off, s[0:3], s33 offset:56 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v4, off, s[0:3], s33 offset:28 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v5, off, s[0:3], s33 offset:32 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v6, off, s[0:3], s33 offset:36 ; 4-byte Folded Reload
+; GFX908-NEXT: buffer_load_dword v7, off, s[0:3], s33 offset:40 ; 4-byte Folded Reload
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: s_waitcnt vmcnt(33)
; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0x70, v2
; GFX908-NEXT: s_waitcnt vmcnt(32)
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: s_waitcnt vmcnt(28)
; GFX908-NEXT: flat_store_dwordx4 v[0:1], v[32:35]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0x60, v2
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 5b232348c4577..00ae3d5b85611 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -531,9 +531,9 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: s_waitcnt vmcnt(2)
; GFX900-NEXT: v_add_co_u32_e32 v16, vcc, v14, v4
; GFX900-NEXT: v_addc_co_u32_e32 v17, vcc, v15, v5, vcc
-; GFX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
; GFX900-NEXT: global_load_dwordx2 v[14:15], v[8:9], off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
+; GFX900-NEXT: s_waitcnt vmcnt(1)
; GFX900-NEXT: v_add_co_u32_e32 v14, vcc, v14, v16
; GFX900-NEXT: v_addc_co_u32_e32 v15, vcc, v15, v17, vcc
; GFX900-NEXT: v_add_co_u32_e32 v8, vcc, s2, v2
@@ -744,15 +744,15 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: ; Parent Loop BB1_1 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_add_co_u32_e64 v18, s[0:1], s3, v6
+; GFX90A-NEXT: v_add_co_u32_e32 v8, vcc, 0xffffe000, v6
; GFX90A-NEXT: v_addc_co_u32_e64 v19, s[0:1], 0, v7, s[0:1]
; GFX90A-NEXT: v_add_co_u32_e64 v20, s[0:1], s4, v6
-; GFX90A-NEXT: v_add_co_u32_e32 v8, vcc, 0xffffe000, v6
-; GFX90A-NEXT: v_addc_co_u32_e64 v21, s[0:1], 0, v7, s[0:1]
; GFX90A-NEXT: v_addc_co_u32_e32 v9, vcc, -1, v7, vcc
-; GFX90A-NEXT: global_load_dwordx2 v[24:25], v[20:21], off offset:-4096
-; GFX90A-NEXT: global_load_dwordx2 v[26:27], v[20:21], off
+; GFX90A-NEXT: v_addc_co_u32_e64 v21, s[0:1], 0, v7, s[0:1]
; GFX90A-NEXT: global_load_dwordx2 v[28:29], v[8:9], off offset:-2048
; GFX90A-NEXT: global_load_dwordx2 v[30:31], v[8:9], off
+; GFX90A-NEXT: global_load_dwordx2 v[24:25], v[20:21], off offset:-4096
+; GFX90A-NEXT: global_load_dwordx2 v[26:27], v[20:21], off
; GFX90A-NEXT: v_add_co_u32_e32 v22, vcc, s2, v6
; GFX90A-NEXT: v_addc_co_u32_e32 v23, vcc, -1, v7, vcc
; GFX90A-NEXT: global_load_dwordx2 v[8:9], v[22:23], off offset:-2048
@@ -769,10 +769,10 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
; GFX90A-NEXT: s_addk_i32 s5, 0x2000
; GFX90A-NEXT: s_cmp_gt_u32 s5, 0x3fffff
-; GFX90A-NEXT: s_waitcnt vmcnt(8)
+; GFX90A-NEXT: s_waitcnt vmcnt(10)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v28, v4
; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v29, v5, vcc
-; GFX90A-NEXT: s_waitcnt vmcnt(7)
+; GFX90A-NEXT: s_waitcnt vmcnt(9)
; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, v30, v4
; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, v31, v5, vcc
; GFX90A-NEXT: s_waitcnt vmcnt(6)
@@ -2184,9 +2184,9 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
; GFX9-NEXT: s_movk_i32 s0, 0x1000
; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, s0, v0
; GFX9-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v1, vcc
-; GFX9-NEXT: global_load_dwordx2 v[14:15], v[12:13], off
; GFX9-NEXT: global_load_dwordx2 v[16:17], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[18:19], v[12:13], off offset:2048
+; GFX9-NEXT: global_load_dwordx2 v[14:15], v[12:13], off
; GFX9-NEXT: global_load_dwordx2 v[20:21], v[0:1], off offset:2048
; GFX9-NEXT: s_waitcnt vmcnt(6)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v6, v4
@@ -2197,12 +2197,13 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v10, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v11, v1, vcc
-; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v16, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v17, v1, vcc
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v18, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v19, v1, vcc
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v14, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v15, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2325,8 +2326,8 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
; GFX11-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v1, vcc_lo
; GFX11-NEXT: s_clause 0x4
; GFX11-NEXT: global_load_b64 v[12:13], v[8:9], off offset:2048
-; GFX11-NEXT: global_load_b64 v[14:15], v[10:11], off
; GFX11-NEXT: global_load_b64 v[8:9], v[8:9], off
+; GFX11-NEXT: global_load_b64 v[14:15], v[10:11], off
; GFX11-NEXT: global_load_b64 v[10:11], v[10:11], off offset:2048
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:2048
; GFX11-NEXT: s_waitcnt vmcnt(6)
@@ -2341,7 +2342,7 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v12, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, v13, v3, vcc_lo
-; GFX11-NEXT: s_waitcnt vmcnt(2)
+; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v8, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, v9, v3, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index e9a48dc5c2af4..b51ac0dd7255b 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -596,47 +596,52 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_9
; GFX9-O0-NEXT: .LBB0_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 1
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
; GFX9-O0-NEXT: s_mov_b32 s4, 63
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8
; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
@@ -656,26 +661,29 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_4
; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while
@@ -686,35 +694,36 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v31, 9
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8
; GFX9-O0-NEXT: s_mov_b32 s5, 1
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
@@ -724,6 +733,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6
; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
@@ -733,41 +743,46 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_or_b32_e64 v7, v3, v6
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
; GFX9-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v28
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v26
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v11, v16
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v27
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v25
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v11
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v23
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v4
; GFX9-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13
; GFX9-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
@@ -784,13 +799,17 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v20
; GFX9-O0-NEXT: v_and_b32_e64 v16, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_and_b32_e64 v19, v11, v19
; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v22
; GFX9-O0-NEXT: v_and_b32_e64 v8, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v21
; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v16
; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
@@ -809,15 +828,19 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17
; GFX9-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18
; GFX9-O0-NEXT: v_mov_b32_e32 v11, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9
; GFX9-O0-NEXT: v_mov_b32_e32 v17, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v10, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
@@ -899,29 +922,30 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
@@ -965,17 +989,21 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
@@ -1003,10 +1031,11 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -1014,40 +1043,44 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_6
; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s4, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3
; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
; GFX9-O0-NEXT: s_mov_b32 s6, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s9, s7
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
@@ -1074,10 +1107,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v12, s10, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v13
; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v14
@@ -1159,6 +1194,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execz .LBB0_5
; GFX9-O0-NEXT: s_branch .LBB0_7
; GFX9-O0-NEXT: .LBB0_9: ; %udiv-end
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
@@ -1171,24 +1210,22 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0xffffffff
; GFX9-O0-NEXT: s_mov_b32 s6, s5
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v17
; GFX9-O0-NEXT: v_and_b32_e64 v3, v0, s6
; GFX9-O0-NEXT: s_mov_b32 s5, s4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v16
; GFX9-O0-NEXT: v_and_b32_e64 v0, v2, s5
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v14
; GFX9-O0-NEXT: v_and_b32_e64 v0, v0, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v18, v13
; GFX9-O0-NEXT: v_and_b32_e64 v23, v18, s5
; GFX9-O0-NEXT: ; kill: def $vgpr23 killed $vgpr23 def $vgpr23_vgpr24 killed $exec
@@ -1328,8 +1365,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v17, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v21
; GFX9-O0-NEXT: v_mul_lo_u32 v14, v13, v15
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[21:22], s4, v[21:22]
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v21
; GFX9-O0-NEXT: v_mul_lo_u32 v13, v2, v13
@@ -1935,47 +1974,52 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_9
; GFX9-O0-NEXT: .LBB1_4: ; %udiv-loop-exit
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 1
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s4, v[0:1]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshlrev_b64 v[9:10], s4, v[9:10]
; GFX9-O0-NEXT: s_mov_b32 s4, 63
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v8
; GFX9-O0-NEXT: v_or3_b32 v4, v4, v11, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v7
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
@@ -1995,26 +2039,29 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v31, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v31, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
@@ -2025,35 +2072,36 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v31, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v31, 9
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(22)
; GFX9-O0-NEXT: v_lshrrev_b64 v[7:8], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v8
; GFX9-O0-NEXT: s_mov_b32 s5, 1
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: v_lshlrev_b64 v[14:15], s5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v15
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
@@ -2063,6 +2111,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v6
; GFX9-O0-NEXT: v_lshlrev_b64 v[29:30], s5, v[2:3]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[27:28]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v30
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
@@ -2072,41 +2121,46 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_or_b32_e64 v7, v3, v6
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshlrev_b64 v[2:3], s5, v[0:1]
; GFX9-O0-NEXT: v_lshlrev_b64 v[27:28], s5, v[27:28]
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v28
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v26
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v11, v16
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v27
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v25
; GFX9-O0-NEXT: v_or3_b32 v0, v0, v1, v11
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v3
-; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v24
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v23
; GFX9-O0-NEXT: v_or_b32_e64 v2, v2, v3
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v7
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v4
; GFX9-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-O0-NEXT: v_subb_co_u32_e64 v4, s[8:9], v4, v7, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v12
; GFX9-O0-NEXT: v_subb_co_u32_e64 v11, s[8:9], v5, v4, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v15
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v13
; GFX9-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v5, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr11 killed $vgpr11 def $vgpr11_vgpr12 killed $exec
@@ -2123,13 +2177,17 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v20
; GFX9-O0-NEXT: v_and_b32_e64 v16, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_and_b32_e64 v19, v11, v19
; GFX9-O0-NEXT: ; kill: def $vgpr19 killed $vgpr19 def $vgpr19_vgpr20 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v22
; GFX9-O0-NEXT: v_and_b32_e64 v8, v8, v16
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v21
; GFX9-O0-NEXT: v_and_b32_e64 v21, v11, v16
; GFX9-O0-NEXT: ; kill: def $vgpr21 killed $vgpr21 def $vgpr21_vgpr22 killed $exec
@@ -2148,15 +2206,19 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v8
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v17
; GFX9-O0-NEXT: v_add_co_u32_e64 v16, s[4:5], v8, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v18
; GFX9-O0-NEXT: v_mov_b32_e32 v11, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v8, v11, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v9
; GFX9-O0-NEXT: v_mov_b32_e32 v17, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v19, s[4:5], v11, v17, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v10, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v9, s[4:5], v9, v10, s[4:5]
@@ -2238,29 +2300,30 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[18:19]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
@@ -2304,17 +2367,21 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
; GFX9-O0-NEXT: s_mov_b32 s8, s11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
@@ -2342,10 +2409,11 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -2353,40 +2421,44 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(20)
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s4, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v3
; GFX9-O0-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, s4
; GFX9-O0-NEXT: s_mov_b32 s6, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_addc_co_u32_e64 v3, s[4:5], v4, v1, s[4:5]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(5)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v12
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v8, s[4:5], v1, v5, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s9, s7
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s9
; GFX9-O0-NEXT: v_addc_co_u32_e64 v1, s[4:5], v1, v5, s[4:5]
@@ -2413,10 +2485,12 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr2_vgpr3 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: v_lshlrev_b64 v[4:5], v2, v[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v12, s10, v2
+; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_lshrrev_b64 v[12:13], v12, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v13
; GFX9-O0-NEXT: v_or_b32_e64 v3, v3, v14
@@ -2498,6 +2572,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_5
; GFX9-O0-NEXT: s_branch .LBB1_7
; GFX9-O0-NEXT: .LBB1_9: ; %udiv-end
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -2506,24 +2584,22 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0xffffffff
; GFX9-O0-NEXT: s_mov_b32 s6, s5
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v13
; GFX9-O0-NEXT: v_and_b32_e64 v4, v0, s6
; GFX9-O0-NEXT: s_mov_b32 s5, s4
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v12
; GFX9-O0-NEXT: v_and_b32_e64 v0, v7, s5
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v9
; GFX9-O0-NEXT: v_and_b32_e64 v0, v0, s6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v8
; GFX9-O0-NEXT: v_and_b32_e64 v10, v14, s5
; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
@@ -2663,8 +2739,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_addc_co_u32_e64 v10, s[6:7], v10, v11, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v10
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v17
; GFX9-O0-NEXT: v_mul_lo_u32 v10, v9, v11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[17:18], s4, v[17:18]
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v17
; GFX9-O0-NEXT: v_mul_lo_u32 v9, v7, v9
diff --git a/llvm/test/CodeGen/AMDGPU/reschedule-bundle-loads.mir b/llvm/test/CodeGen/AMDGPU/reschedule-bundle-loads.mir
new file mode 100644
index 0000000000000..f6c6f2d0fb77e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/reschedule-bundle-loads.mir
@@ -0,0 +1,198 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -start-before si-post-ra-bundler -o - %s | FileCheck %s
+
+--- |
+ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+ target triple = "amdgcn-amd-amdpal"
+ define dllexport amdgpu_cs void @_amdgpu_cs_main(ptr inreg noundef %userdata2, ptr %out0, ptr %out1, ptr %out2, ptr %out3) {
+ ; CHECK-LABEL: _amdgpu_cs_main:
+ ; CHECK: ; %bb.0: ; %.entry
+ ; CHECK-NEXT: ; implicit-def: $vgpr11
+ ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; CHECK-NEXT: v_cvt_u32_f32_e32 v15, v11
+ ; CHECK-NEXT: ; implicit-def: $vgpr12
+ ; CHECK-NEXT: v_cvt_u32_f32_e32 v28, v12
+ ; CHECK-NEXT: ; implicit-def: $vgpr16
+ ; CHECK-NEXT: ; implicit-def: $vgpr17
+ ; CHECK-NEXT: ; implicit-def: $vgpr19
+ ; CHECK-NEXT: ; implicit-def: $vgpr22
+ ; CHECK-NEXT: ; implicit-def: $vgpr24
+ ; CHECK-NEXT: ; implicit-def: $vgpr25
+ ; CHECK-NEXT: ; implicit-def: $vgpr26
+ ; CHECK-NEXT: ; implicit-def: $vgpr57
+ ; CHECK-NEXT: ; implicit-def: $vgpr58
+ ; CHECK-NEXT: ; implicit-def: $vgpr59
+ ; CHECK-NEXT: ; implicit-def: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+ ; CHECK-NEXT: ; implicit-def: $vgpr6
+ ; CHECK-NEXT: v_lshlrev_b32_e32 v56, 3, v6
+ ; CHECK-NEXT: s_clause 0x1f
+ ; CHECK-NEXT: image_load v9, [v57, v58, v19], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v10, [v57, v58, v17], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v12, [v15, v58, v19], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v36, [v15, v58, v17], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v41, [v57, v28, v19], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v42, [v57, v28, v17], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v47, [v15, v28, v19], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v48, [v15, v28, v17], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v11, [v57, v58, v22], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v51, [v57, v58, v16], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v35, [v15, v58, v22], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v52, [v15, v58, v16], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v49, [v57, v28, v22], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v53, [v57, v28, v16], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v50, [v15, v28, v22], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v54, [v15, v28, v16], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v20, v[57:59], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v21, [v57, v58, v24], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v31, [v15, v58, v59], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v32, [v15, v58, v24], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v37, [v57, v28, v59], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v38, [v57, v28, v24], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v43, [v15, v28, v59], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v44, [v15, v28, v24], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v29, [v57, v58, v25], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v30, [v57, v58, v26], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v33, [v15, v58, v25], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v34, [v15, v58, v26], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v39, [v57, v28, v25], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v40, [v57, v28, v26], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v45, [v15, v28, v25], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: image_load v46, [v15, v28, v26], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm
+ ; CHECK-NEXT: ; implicit-def: $vgpr7
+ ; CHECK-NEXT: ; implicit-def: $vgpr0
+ ; CHECK-NEXT: ; implicit-def: $vgpr3
+ ; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v7
+ ; CHECK-NEXT: s_waitcnt vmcnt(30)
+ ; CHECK-NEXT: v_max3_f32 v55, v9, 0, v10
+ ; CHECK-NEXT: v_lshrrev_b32_e32 v9, 10, v0
+ ; CHECK-NEXT: s_waitcnt vmcnt(28)
+ ; CHECK-NEXT: v_max3_f32 v36, v12, 0, v36
+ ; CHECK-NEXT: v_lshlrev_b32_e32 v12, 1, v7
+ ; CHECK-NEXT: s_waitcnt vmcnt(26)
+ ; CHECK-NEXT: v_max3_f32 v41, v41, 0, v42
+ ; CHECK-NEXT: v_lshlrev_b32_e32 v10, 1, v6
+ ; CHECK-NEXT: s_waitcnt vmcnt(24)
+ ; CHECK-NEXT: v_max3_f32 v42, v47, 0, v48
+ ; CHECK-NEXT: s_waitcnt vmcnt(22)
+ ; CHECK-NEXT: v_max3_f32 v47, v55, v11, v51
+ ; CHECK-NEXT: v_mad_u32_u24 v11, 0x90, v7, v56
+ ; CHECK-NEXT: s_waitcnt vmcnt(20)
+ ; CHECK-NEXT: v_max3_f32 v35, v36, v35, v52
+ ; CHECK-NEXT: s_waitcnt vmcnt(18)
+ ; CHECK-NEXT: v_max3_f32 v36, v41, v49, v53
+ ; CHECK-NEXT: s_waitcnt vmcnt(16)
+ ; CHECK-NEXT: v_max3_f32 v41, v42, v50, v54
+ ; CHECK-NEXT: s_waitcnt vmcnt(14)
+ ; CHECK-NEXT: v_max3_f32 v20, v47, v20, v21
+ ; CHECK-NEXT: v_add_nc_u32_e32 v21, 2, v3
+ ; CHECK-NEXT: s_waitcnt vmcnt(12)
+ ; CHECK-NEXT: v_max3_f32 v31, v35, v31, v32
+ ; CHECK-NEXT: s_waitcnt vmcnt(10)
+ ; CHECK-NEXT: v_max3_f32 v32, v36, v37, v38
+ ; CHECK-NEXT: s_waitcnt vmcnt(8)
+ ; CHECK-NEXT: v_max3_f32 v35, v41, v43, v44
+ ; CHECK-NEXT: s_waitcnt vmcnt(6)
+ ; CHECK-NEXT: v_max3_f32 v29, v20, v29, v30
+ ; CHECK-NEXT: v_add_nc_u32_e32 v20, 3, v3
+ ; CHECK-NEXT: s_waitcnt vmcnt(4)
+ ; CHECK-NEXT: v_max3_f32 v30, v31, v33, v34
+ ; CHECK-NEXT: s_waitcnt vmcnt(2)
+ ; CHECK-NEXT: v_max3_f32 v31, v32, v39, v40
+ ; CHECK-NEXT: s_waitcnt vmcnt(0)
+ ; CHECK-NEXT: v_max3_f32 v32, v35, v45, v46
+ ; CHECK-NEXT: ds_store_2addr_b32 v11, v29, v31 offset1:1
+ ; CHECK-NEXT: ds_store_2addr_b32 v11, v30, v32 offset0:18 offset1:19
+ ; CHECK-NEXT: s_endpgm
+ .entry:
+ ret void
+ }
+
+...
+---
+name: _amdgpu_cs_main
+exposesReturnsTwice: false
+tracksRegLiveness: true
+body: |
+ bb.0..entry:
+
+ $vgpr0 = IMPLICIT_DEF
+ $vgpr3 = IMPLICIT_DEF
+ $vgpr6 = IMPLICIT_DEF
+ $vgpr7 = IMPLICIT_DEF
+ $vgpr11 = IMPLICIT_DEF
+ $vgpr12 = IMPLICIT_DEF
+ $vgpr16 = IMPLICIT_DEF
+ $vgpr17 = IMPLICIT_DEF
+ $vgpr19 = IMPLICIT_DEF
+ $vgpr22 = IMPLICIT_DEF
+ $vgpr24 = IMPLICIT_DEF
+ $vgpr25 = IMPLICIT_DEF
+ $vgpr26 = IMPLICIT_DEF
+ $vgpr57 = IMPLICIT_DEF
+ $vgpr58 = IMPLICIT_DEF
+ $vgpr59 = IMPLICIT_DEF
+ $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = IMPLICIT_DEF
+
+ renamable $vgpr15 = nofpexcept V_CVT_U32_F32_e32 killed $vgpr11, implicit $mode, implicit $exec
+ renamable $vgpr28 = nofpexcept V_CVT_U32_F32_e32 killed $vgpr12, implicit $mode, implicit $exec
+ renamable $vgpr20 = IMAGE_LOAD_V1_V3_gfx11 $vgpr57_vgpr58_vgpr59, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr21 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr24, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr29 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr25, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr30 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr26, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr11 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr22, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr9 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr19, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr10 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr17, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr31 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr59, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr32 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr24, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr33 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr25, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr34 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr26, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr35 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr22, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr12 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr19, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr36 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr17, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr37 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr59, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr38 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr24, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr39 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr25, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr40 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr26, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr41 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr19, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr42 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr17, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr43 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr59, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr44 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr24, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr45 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr25, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr46 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr26, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr47 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr19, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr48 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr17, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr49 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr22, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr50 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr22, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr51 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr58, renamable $vgpr16, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr52 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr58, renamable $vgpr16, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr53 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr57, renamable $vgpr28, renamable $vgpr16, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr54 = IMAGE_LOAD_V1_V3_nsa_gfx11 renamable $vgpr15, renamable $vgpr28, renamable $vgpr16, renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, 1, 6, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
+ renamable $vgpr55 = V_MAX3_F32_e64 0, killed $vgpr9, 0, 0, 0, killed $vgpr10, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr9 = V_LSHRREV_B32_e32 10, $vgpr0, implicit $exec
+ renamable $vgpr36 = V_MAX3_F32_e64 0, killed $vgpr12, 0, 0, 0, killed $vgpr36, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr56 = V_LSHLREV_B32_e32 3, $vgpr6, implicit $exec
+ renamable $vgpr41 = V_MAX3_F32_e64 0, killed $vgpr41, 0, 0, 0, killed $vgpr42, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr12 = nuw nsw V_LSHLREV_B32_e32 1, $vgpr7, implicit $exec
+ renamable $vgpr42 = V_MAX3_F32_e64 0, killed $vgpr47, 0, 0, 0, killed $vgpr48, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr10 = nuw nsw V_LSHLREV_B32_e32 1, $vgpr6, implicit $exec
+ V_CMP_EQ_U32_e32 7, $vgpr7, implicit-def $vcc, implicit $exec
+ renamable $vgpr47 = V_MAX3_F32_e64 0, killed $vgpr55, 0, killed $vgpr11, 0, killed $vgpr51, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr35 = V_MAX3_F32_e64 0, killed $vgpr36, 0, killed $vgpr35, 0, killed $vgpr52, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr36 = V_MAX3_F32_e64 0, killed $vgpr41, 0, killed $vgpr49, 0, killed $vgpr53, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr41 = V_MAX3_F32_e64 0, killed $vgpr42, 0, killed $vgpr50, 0, killed $vgpr54, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr11 = V_MAD_U32_U24_e64 144, $vgpr7, killed $vgpr56, 0, implicit $exec
+ renamable $vgpr20 = V_MAX3_F32_e64 0, killed $vgpr47, 0, killed $vgpr20, 0, killed $vgpr21, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr31 = V_MAX3_F32_e64 0, killed $vgpr35, 0, killed $vgpr31, 0, killed $vgpr32, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr32 = V_MAX3_F32_e64 0, killed $vgpr36, 0, killed $vgpr37, 0, killed $vgpr38, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr35 = V_MAX3_F32_e64 0, killed $vgpr41, 0, killed $vgpr43, 0, killed $vgpr44, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr21 = V_ADD_U32_e32 2, $vgpr3, implicit $exec
+ renamable $vgpr29 = V_MAX3_F32_e64 0, killed $vgpr20, 0, killed $vgpr29, 0, killed $vgpr30, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr30 = V_MAX3_F32_e64 0, killed $vgpr31, 0, killed $vgpr33, 0, killed $vgpr34, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr31 = V_MAX3_F32_e64 0, killed $vgpr32, 0, killed $vgpr39, 0, killed $vgpr40, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr32 = V_MAX3_F32_e64 0, killed $vgpr35, 0, killed $vgpr45, 0, killed $vgpr46, 0, 0, implicit $mode, implicit $exec
+ renamable $vgpr20 = V_ADD_U32_e32 3, $vgpr3, implicit $exec
+ DS_WRITE2_B32_gfx9 renamable $vgpr11, killed renamable $vgpr29, killed renamable $vgpr31, 0, 1, 0, implicit $exec :: (store (s32) into %ir.out0, addrspace 3), (store (s32) into %ir.out1, addrspace 3)
+ DS_WRITE2_B32_gfx9 renamable $vgpr11, killed renamable $vgpr30, killed renamable $vgpr32, 18, 19, 0, implicit $exec :: (store (s32) into %ir.out2, addrspace 3), (store (s32) into %ir.out3, addrspace 3)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index d4ff8ed56b7c6..b69ab693915c4 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -2266,8 +2266,8 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v32
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:92
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:84
; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v33
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88
@@ -2303,10 +2303,10 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[8:9]
; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_eq_u32_e64 s[8:9], 0, v51
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:36
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:44
; SI-NEXT: v_cndmask_b32_e64 v28, v27, v11, s[8:9]
-; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:44
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:36
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_cmp_eq_u32_e64 s[8:9], 0, v32
; SI-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[8:9]
; SI-NEXT: s_waitcnt vmcnt(7)
@@ -2315,8 +2315,8 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: v_cmp_eq_u32_e64 s[8:9], 0, v31
; SI-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[8:9]
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:48
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:40
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:32
; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:24
; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:16
; SI-NEXT: s_waitcnt vmcnt(11)
@@ -2330,8 +2330,8 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: s_waitcnt vmcnt(10)
; SI-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v36
; SI-NEXT: v_cndmask_b32_e64 v8, v24, v8, s[6:7]
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v38
; SI-NEXT: v_cndmask_b32_e64 v38, v23, v7, s[6:7]
@@ -2358,11 +2358,12 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: v_cndmask_b32_e32 v26, v21, v5, vcc
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v49
; SI-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v32
; SI-NEXT: v_cndmask_b32_e32 v21, v20, v4, vcc
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29
; SI-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31
; SI-NEXT: v_cndmask_b32_e32 v20, v19, v3, vcc
; SI-NEXT: s_waitcnt vmcnt(4)
@@ -2370,11 +2371,12 @@ define <32 x half> @v_vselect_v32f16(<32 x half> %a, <32 x half> %b, <32 x i32>
; SI-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v33
; SI-NEXT: v_cndmask_b32_e32 v19, v18, v2, vcc
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v36
; SI-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v51
; SI-NEXT: v_cndmask_b32_e32 v18, v17, v1, vcc
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24
; SI-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc
; SI-NEXT: s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
index 540f4fcfc6d67..2ec12e988d146 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
@@ -260,16 +260,16 @@ define amdgpu_kernel void @v_shl_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GCN-NEXT: v_lshlrev_b32_e32 v16, 5, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x3
-; GCN-NEXT: global_load_b128 v[0:3], v16, s[2:3] offset:48
; GCN-NEXT: global_load_b128 v[4:7], v16, s[2:3] offset:32
; GCN-NEXT: global_load_b128 v[8:11], v16, s[2:3]
+; GCN-NEXT: global_load_b128 v[0:3], v16, s[2:3] offset:48
; GCN-NEXT: global_load_b128 v[12:15], v16, s[2:3] offset:16
-; GCN-NEXT: s_wait_loadcnt 0x1
+; GCN-NEXT: s_wait_loadcnt 0x2
; GCN-NEXT: v_lshlrev_b64_e32 v[6:7], v6, v[10:11]
+; GCN-NEXT: v_lshlrev_b64_e32 v[4:5], v4, v[8:9]
; GCN-NEXT: s_wait_loadcnt 0x0
; GCN-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[14:15]
; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[12:13]
-; GCN-NEXT: v_lshlrev_b64_e32 v[4:5], v4, v[8:9]
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:16
; GCN-NEXT: global_store_b128 v16, v[4:7], s[0:1]
@@ -332,26 +332,26 @@ define amdgpu_kernel void @v_shl_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GCN-NEXT: v_lshlrev_b32_e32 v32, 6, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_load_b128 v[0:3], v32, s[2:3]
; GCN-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16
-; GCN-NEXT: global_load_b128 v[8:11], v32, s[2:3] offset:64
; GCN-NEXT: global_load_b128 v[12:15], v32, s[2:3] offset:80
-; GCN-NEXT: global_load_b128 v[16:19], v32, s[2:3] offset:32
+; GCN-NEXT: global_load_b128 v[0:3], v32, s[2:3]
+; GCN-NEXT: global_load_b128 v[8:11], v32, s[2:3] offset:64
; GCN-NEXT: global_load_b128 v[20:23], v32, s[2:3] offset:48
; GCN-NEXT: global_load_b128 v[24:27], v32, s[2:3] offset:112
+; GCN-NEXT: global_load_b128 v[16:19], v32, s[2:3] offset:32
; GCN-NEXT: global_load_b128 v[28:31], v32, s[2:3] offset:96
-; GCN-NEXT: s_wait_loadcnt 0x5
-; GCN-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3]
-; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1]
-; GCN-NEXT: s_wait_loadcnt 0x4
+; GCN-NEXT: s_wait_loadcnt 0x6
; GCN-NEXT: v_lshlrev_b64_e32 v[6:7], v14, v[6:7]
; GCN-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[4:5]
-; GCN-NEXT: s_wait_loadcnt 0x1
+; GCN-NEXT: s_wait_loadcnt 0x4
+; GCN-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3]
+; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1]
+; GCN-NEXT: s_wait_loadcnt 0x2
; GCN-NEXT: v_lshlrev_b64_e32 v[22:23], v26, v[22:23]
+; GCN-NEXT: v_lshlrev_b64_e32 v[20:21], v24, v[20:21]
; GCN-NEXT: s_wait_loadcnt 0x0
; GCN-NEXT: v_lshlrev_b64_e32 v[18:19], v30, v[18:19]
; GCN-NEXT: v_lshlrev_b64_e32 v[16:17], v28, v[16:17]
-; GCN-NEXT: v_lshlrev_b64_e32 v[20:21], v24, v[20:21]
; GCN-NEXT: s_clause 0x3
; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:32
; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48
@@ -388,13 +388,13 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1250-NEXT: global_load_b128 v[12:15], v52, s[2:3] offset:128
; GFX1250-NEXT: global_load_b128 v[16:19], v52, s[2:3] offset:64
; GFX1250-NEXT: global_load_b128 v[20:23], v52, s[2:3] offset:192
-; GFX1250-NEXT: global_load_b128 v[24:27], v52, s[2:3] offset:96
; GFX1250-NEXT: global_load_b128 v[28:31], v52, s[2:3] offset:112
-; GFX1250-NEXT: global_load_b128 v[32:35], v52, s[2:3] offset:80
-; GFX1250-NEXT: global_load_b128 v[36:39], v52, s[2:3] offset:32
+; GFX1250-NEXT: global_load_b128 v[48:51], v52, s[2:3] offset:240
; GFX1250-NEXT: global_load_b128 v[40:43], v52, s[2:3] offset:48
+; GFX1250-NEXT: global_load_b128 v[24:27], v52, s[2:3] offset:96
; GFX1250-NEXT: global_load_b128 v[44:47], v52, s[2:3] offset:224
-; GFX1250-NEXT: global_load_b128 v[48:51], v52, s[2:3] offset:240
+; GFX1250-NEXT: global_load_b128 v[36:39], v52, s[2:3] offset:32
+; GFX1250-NEXT: global_load_b128 v[32:35], v52, s[2:3] offset:80
; GFX1250-NEXT: s_wait_loadcnt 0xb
; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[6:7]
; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[4:5]
@@ -407,10 +407,9 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1250-NEXT: v_lshlrev_b64_e32 v[18:19], v22, v[18:19]
; GFX1250-NEXT: v_lshlrev_b64_e32 v[16:17], v20, v[16:17]
; GFX1250-NEXT: global_load_b128 v[20:23], v52, s[2:3] offset:160
-; GFX1250-NEXT: s_wait_loadcnt 0x4
+; GFX1250-NEXT: s_wait_loadcnt 0x5
; GFX1250-NEXT: v_lshlrev_b64_e32 v[26:27], v46, v[26:27]
; GFX1250-NEXT: v_lshlrev_b64_e32 v[24:25], v44, v[24:25]
-; GFX1250-NEXT: s_wait_loadcnt 0x3
; GFX1250-NEXT: v_lshlrev_b64_e32 v[30:31], v50, v[30:31]
; GFX1250-NEXT: v_lshlrev_b64_e32 v[28:29], v48, v[28:29]
; GFX1250-NEXT: s_wait_loadcnt 0x2
@@ -450,18 +449,18 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1251-NEXT: global_load_b128 v[12:15], v52, s[2:3] offset:128
; GFX1251-NEXT: global_load_b128 v[16:19], v52, s[2:3] offset:176
; GFX1251-NEXT: global_load_b128 v[20:23], v52, s[2:3] offset:48
+; GFX1251-NEXT: global_load_b128 v[36:39], v52, s[2:3] offset:112
; GFX1251-NEXT: global_load_b128 v[24:27], v52, s[2:3] offset:32
; GFX1251-NEXT: global_load_b128 v[28:31], v52, s[2:3] offset:160
; GFX1251-NEXT: global_load_b128 v[32:35], v52, s[2:3] offset:96
-; GFX1251-NEXT: global_load_b128 v[36:39], v52, s[2:3] offset:112
-; GFX1251-NEXT: global_load_b128 v[40:43], v52, s[2:3] offset:64
; GFX1251-NEXT: global_load_b128 v[44:47], v52, s[2:3] offset:80
+; GFX1251-NEXT: global_load_b128 v[40:43], v52, s[2:3] offset:64
; GFX1251-NEXT: s_wait_loadcnt 0xa
; GFX1251-NEXT: v_lshlrev_b64_e32 v[2:3], v2, v[6:7]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[0:1], v0, v[4:5]
; GFX1251-NEXT: s_clause 0x1
-; GFX1251-NEXT: global_load_b128 v[4:7], v52, s[2:3] offset:224
; GFX1251-NEXT: global_load_b128 v[48:51], v52, s[2:3] offset:240
+; GFX1251-NEXT: global_load_b128 v[4:7], v52, s[2:3] offset:224
; GFX1251-NEXT: s_wait_loadcnt 0xa
; GFX1251-NEXT: v_lshlrev_b64_e32 v[10:11], v14, v[10:11]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[8:9], v12, v[8:9]
@@ -471,14 +470,14 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: global_load_b128 v[16:19], v52, s[2:3] offset:208
; GFX1251-NEXT: global_load_b128 v[20:23], v52, s[2:3] offset:192
-; GFX1251-NEXT: s_wait_loadcnt 0x8
+; GFX1251-NEXT: s_wait_loadcnt 0x7
; GFX1251-NEXT: v_lshlrev_b64_e32 v[26:27], v30, v[26:27]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[24:25], v28, v[24:25]
; GFX1251-NEXT: s_wait_loadcnt 0x3
+; GFX1251-NEXT: v_lshlrev_b64_e32 v[38:39], v50, v[38:39]
+; GFX1251-NEXT: s_wait_loadcnt 0x2
; GFX1251-NEXT: v_lshlrev_b64_e32 v[6:7], v6, v[34:35]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[4:5], v4, v[32:33]
-; GFX1251-NEXT: s_wait_loadcnt 0x2
-; GFX1251-NEXT: v_lshlrev_b64_e32 v[38:39], v50, v[38:39]
; GFX1251-NEXT: v_lshlrev_b64_e32 v[36:37], v48, v[36:37]
; GFX1251-NEXT: s_wait_loadcnt 0x1
; GFX1251-NEXT: v_lshlrev_b64_e32 v[18:19], v18, v[46:47]
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 5c829d7ec88ef..082d3120452f1 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -9866,11 +9866,11 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: buffer_store_dword v1, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v2, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v3, off, s[40:43], s2 offset:12 ; 4-byte Folded Spill
+; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v[5:6], s[4:7], 0 addr64
-; GFX6-NEXT: buffer_load_dwordx4 v[7:10], v[5:6], s[4:7], 0 addr64 offset:16
; GFX6-NEXT: s_mov_b32 s2, 0x83000
-; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: s_waitcnt vmcnt(1)
; GFX6-NEXT: buffer_store_dword v7, off, s[40:43], s2 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v8, off, s[40:43], s2 offset:4 ; 4-byte Folded Spill
; GFX6-NEXT: buffer_store_dword v9, off, s[40:43], s2 offset:8 ; 4-byte Folded Spill
@@ -10301,11 +10301,11 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2030
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:160
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[19:22], v5, s[38:39] offset:192
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[15:18], v5, s[38:39] offset:176
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:160
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2020
-; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(2)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x2070
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[6:9], s0 ; 16-byte Folded Spill
@@ -10319,8 +10319,8 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:96
; GFX9-FLATSCR-NEXT: s_movk_i32 s0, 0x20b0
-; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:32
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[6:9], v5, s[38:39] offset:16
+; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[11:14], v5, s[38:39] offset:32
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(2)
; GFX9-FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s0 ; 16-byte Folded Spill
; GFX9-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39] offset:80
@@ -10473,6 +10473,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: v_lshlrev_b32_e32 v5, 8, v0
; GFX10-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FLATSCR-NEXT: s_clause 0xf
+; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39]
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[35:38], v5, s[38:39] offset:240
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[31:34], v5, s[38:39] offset:224
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[27:30], v5, s[38:39] offset:208
@@ -10488,8 +10489,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[47:50], v5, s[38:39] offset:48
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[43:46], v5, s[38:39] offset:32
; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[39:42], v5, s[38:39] offset:16
-; GFX10-FLATSCR-NEXT: global_load_dwordx4 v[0:3], v5, s[38:39]
-; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(15)
; GFX10-FLATSCR-NEXT: v_lshl_add_u32 v4, v0, 13, 16
; GFX10-FLATSCR-NEXT: scratch_store_dword v4, v6, off
; GFX10-FLATSCR-NEXT: ;;#ASMSTART
@@ -10520,6 +10520,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: ; use s[0:7],s[8:15],s[16:23],s[24:31],s[40:43],s[34:35]
; GFX10-FLATSCR-NEXT: ;;#ASMEND
; GFX10-FLATSCR-NEXT: s_movk_i32 s0, 0x2010
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(4)
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v88, v58
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v87, v57
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v86, v56
@@ -10541,6 +10542,7 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v35, v9
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v34, v8
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v33, v7
+; GFX10-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v72, v42
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v71, v41
; GFX10-FLATSCR-NEXT: v_mov_b32_e32 v70, v40
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
index c40e35bbe1754..a51821f9f288e 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
@@ -199,13 +199,13 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_and_b32_e32 v0, 1, v2
; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v1
; CHECK-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v0
+; CHECK-NEXT: buffer_load_ubyte v1, off, s[0:3], s32 offset:792
; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:392
; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:388
; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:384
; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:380
; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:376
; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:372
-; CHECK-NEXT: buffer_load_ubyte v1, off, s[0:3], s32 offset:792
; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788
; CHECK-NEXT: s_waitcnt vmcnt(21)
; CHECK-NEXT: v_and_b32_e32 v17, 1, v18
@@ -308,7 +308,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_store_dword a41, off, s[0:3], s32 offset:1228 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a42, off, s[0:3], s32 offset:1232 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a43, off, s[0:3], s32 offset:1236 ; 4-byte Folded Spill
-; CHECK-NEXT: s_waitcnt vmcnt(60)
+; CHECK-NEXT: s_waitcnt vmcnt(59)
; CHECK-NEXT: buffer_store_dword a44, off, s[0:3], s32 offset:1240 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a45, off, s[0:3], s32 offset:1244 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a46, off, s[0:3], s32 offset:1248 ; 4-byte Folded Spill
@@ -418,6 +418,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_store_dword a29, off, s[0:3], s32 offset:1116 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a30, off, s[0:3], s32 offset:1120 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword a31, off, s[0:3], s32 offset:1124 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:444
; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:412
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword a6, off, s[0:3], s32 offset:408
@@ -429,22 +430,21 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:424
; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:440
; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:436
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:444
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_waitcnt vmcnt(10)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:432
-; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:448
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_load_dword a24, off, s[0:3], s32 offset:432
+; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:428
-; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:452
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:428
+; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Spill
@@ -455,22 +455,15 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword a28, off, s[0:3], s32 offset:8
; CHECK-NEXT: buffer_load_dword a26, off, s[0:3], s32 offset:4
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1132 ; 4-byte Folded Spill
-; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:456
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:584
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:580
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:576
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:572
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:568
; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:564
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:560
; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:556
; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:552
; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:548
@@ -481,11 +474,18 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:528
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:524
+; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:456
+; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:584
+; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:580
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:576
+; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:572
+; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:568
+; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:560
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Spill
-; CHECK-NEXT: s_waitcnt vmcnt(2)
+; CHECK-NEXT: s_waitcnt vmcnt(9)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Spill
@@ -535,99 +535,99 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_write_b32 a5, v57
; CHECK-NEXT: v_accvgpr_write_b32 a2, v50
; CHECK-NEXT: v_accvgpr_write_b32 a3, v51
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_write_b32 a16, v58
; CHECK-NEXT: v_accvgpr_write_b32 a17, v59
+; CHECK-NEXT: s_waitcnt vmcnt(62)
; CHECK-NEXT: v_accvgpr_read_b32 v60, a36
; CHECK-NEXT: v_accvgpr_read_b32 v58, a40
; CHECK-NEXT: v_accvgpr_read_b32 v59, a17
; CHECK-NEXT: s_xor_b64 s[54:55], exec, -1
-; CHECK-NEXT: s_waitcnt vmcnt(29)
-; CHECK-NEXT: v_xor_b32_e32 v28, v28, v18
-; CHECK-NEXT: s_waitcnt vmcnt(28)
+; CHECK-NEXT: s_waitcnt vmcnt(31)
; CHECK-NEXT: v_xor_b32_e32 v0, v29, v3
; CHECK-NEXT: v_accvgpr_write_b32 a9, v0
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a27
-; CHECK-NEXT: s_waitcnt vmcnt(26)
-; CHECK-NEXT: v_xor_b32_e32 v29, v31, v29
-; CHECK-NEXT: s_waitcnt vmcnt(24)
+; CHECK-NEXT: s_waitcnt vmcnt(29)
; CHECK-NEXT: v_xor_b32_e32 v0, v33, v7
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a27
; CHECK-NEXT: v_accvgpr_write_b32 a27, v0
+; CHECK-NEXT: s_waitcnt vmcnt(13)
; CHECK-NEXT: v_xor_b32_e32 v0, v30, v4
-; CHECK-NEXT: v_accvgpr_write_b32 a12, v0
-; CHECK-NEXT: s_waitcnt vmcnt(20)
; CHECK-NEXT: v_xor_b32_e32 v62, v37, v11
+; CHECK-NEXT: v_accvgpr_write_b32 a12, v0
+; CHECK-NEXT: s_waitcnt vmcnt(12)
; CHECK-NEXT: v_xor_b32_e32 v0, v32, v6
+; CHECK-NEXT: s_waitcnt vmcnt(11)
; CHECK-NEXT: v_xor_b32_e32 v32, v34, v8
+; CHECK-NEXT: s_waitcnt vmcnt(10)
; CHECK-NEXT: v_xor_b32_e32 v34, v36, v10
; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
+; CHECK-NEXT: v_xor_b32_e32 v29, v31, v29
; CHECK-NEXT: v_accvgpr_read_b32 v31, a39
; CHECK-NEXT: v_accvgpr_read_b32 v30, a46
; CHECK-NEXT: v_xor_b32_e32 v33, v35, v9
-; CHECK-NEXT: s_waitcnt vmcnt(20)
; CHECK-NEXT: v_xor_b32_e32 v35, v39, v13
-; CHECK-NEXT: s_waitcnt vmcnt(15)
+; CHECK-NEXT: v_xor_b32_e32 v13, v53, v31
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a29
+; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: v_xor_b32_e32 v10, v44, v30
; CHECK-NEXT: v_accvgpr_read_b32 v30, a44
-; CHECK-NEXT: s_waitcnt vmcnt(11)
+; CHECK-NEXT: v_xor_b32_e32 v1, v55, v31
+; CHECK-NEXT: s_waitcnt vmcnt(6)
; CHECK-NEXT: v_xor_b32_e32 v6, v48, v30
; CHECK-NEXT: v_accvgpr_read_b32 v30, a42
; CHECK-NEXT: v_xor_b32_e32 v9, v47, v21
-; CHECK-NEXT: s_waitcnt vmcnt(8)
; CHECK-NEXT: v_xor_b32_e32 v21, v51, v25
-; CHECK-NEXT: v_accvgpr_write_b32 a36, v0
-; CHECK-NEXT: s_waitcnt vmcnt(6)
-; CHECK-NEXT: v_xor_b32_e32 v13, v53, v31
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a29
-; CHECK-NEXT: s_waitcnt vmcnt(4)
-; CHECK-NEXT: v_xor_b32_e32 v1, v55, v31
-; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: v_xor_b32_e32 v25, v56, v60
+; CHECK-NEXT: v_accvgpr_write_b32 a36, v0
; CHECK-NEXT: v_xor_b32_e32 v8, v46, v20
+; CHECK-NEXT: s_waitcnt vmcnt(5)
; CHECK-NEXT: v_xor_b32_e32 v20, v50, v30
; CHECK-NEXT: v_accvgpr_read_b32 v30, a38
+; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: v_xor_b32_e32 v0, v54, v58
; CHECK-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1
; CHECK-NEXT: v_xor_b32_e32 v11, v45, v19
; CHECK-NEXT: v_xor_b32_e32 v19, v27, v5
; CHECK-NEXT: v_xor_b32_e32 v4, v52, v30
+; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: v_xor_b32_e32 v30, v26, v2
; CHECK-NEXT: v_lshlrev_b32_e32 v5, 17, v25
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 14, v[0:1]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v5
; CHECK-NEXT: v_and_b32_e32 v5, 0x7fffffff, v13
+; CHECK-NEXT: v_xor_b32_e32 v28, v28, v18
; CHECK-NEXT: v_xor_b32_e32 v18, v38, v12
; CHECK-NEXT: v_lshlrev_b32_e32 v12, 18, v0
; CHECK-NEXT: v_lshrrev_b64 v[0:1], 13, v[4:5]
@@ -638,7 +638,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
; CHECK-NEXT: v_accvgpr_read_b32 v31, a37
-; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: v_xor_b32_e32 v23, v57, v31
; CHECK-NEXT: v_xor_b32_e32 v17, v43, v17
; CHECK-NEXT: v_xor_b32_e32 v16, v42, v16
@@ -903,8 +902,8 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1132 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1128 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_accvgpr_read_b32 v5, a29
; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v5
@@ -997,53 +996,39 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: ; %bb.42: ; %bb0a
; CHECK-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1008 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1004 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:1116 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:1120 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:1124 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:1112 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:1108 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:1104 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:1100 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:1096 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1092 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1088 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1084 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1080 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1076 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1072 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1068 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1064 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1060 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1056 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1052 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1048 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1044 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1040 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1036 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1032 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1028 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1024 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1020 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1016 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1012 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1000 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
-; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
-; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
-; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
-; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
-; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
-; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
-; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
-; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
-; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
-; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
-; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
-; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v36
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v52
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v39
@@ -1058,27 +1043,34 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_or_b32_e32 v9, v1, v0
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 17, v38
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[36:37]
+; CHECK-NEXT: v_and_b32_e32 v35, 0x7fffffff, v35
; CHECK-NEXT: v_or_b32_e32 v5, v5, v0
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v52
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v36
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[34:35]
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v53, vcc
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
+; CHECK-NEXT: v_and_b32_e32 v33, 0x7fffffff, v33
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v32
+; CHECK-NEXT: v_and_b32_e32 v31, 0x7fffffff, v31
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v34
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[32:33]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 20, v32
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 11, v[30:31]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
+; CHECK-NEXT: v_and_b32_e32 v29, 0x7fffffff, v29
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:64
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
+; CHECK-NEXT: v_and_b32_e32 v27, 0x7fffffff, v27
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 21, v30
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 10, v[28:29]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 22, v28
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 9, v[26:27]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
+; CHECK-NEXT: v_and_b32_e32 v25, 0x7fffffff, v25
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:48
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v20
+; CHECK-NEXT: v_and_b32_e32 v23, 0x7fffffff, v23
; CHECK-NEXT: v_lshlrev_b32_e32 v4, 23, v26
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 8, v[24:25]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
@@ -1086,14 +1078,18 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_lshlrev_b32_e32 v4, 24, v24
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 7, v[22:23]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
+; CHECK-NEXT: v_and_b32_e32 v21, 0x7fffffff, v21
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3] offset:32
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 25, v22
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 6, v[20:21]
+; CHECK-NEXT: v_and_b32_e32 v19, 0x7fffffff, v19
; CHECK-NEXT: v_or_b32_e32 v5, v5, v2
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 26, v20
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 5, v[18:19]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
+; CHECK-NEXT: v_and_b32_e32 v17, 0x7fffffff, v17
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:16
-; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
+; CHECK-NEXT: v_and_b32_e32 v15, 0x7fffffff, v15
; CHECK-NEXT: v_lshlrev_b32_e32 v4, 27, v18
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 4, v[16:17]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
@@ -1101,12 +1097,15 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_lshlrev_b32_e32 v4, 28, v16
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 3, v[14:15]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v4
+; CHECK-NEXT: v_and_b32_e32 v13, 0x7fffffff, v13
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 29, v14
; CHECK-NEXT: v_lshrrev_b64 v[0:1], 2, v[12:13]
; CHECK-NEXT: v_or_b32_e32 v1, v1, v2
+; CHECK-NEXT: v_and_b32_e32 v11, 0x7fffffff, v11
; CHECK-NEXT: flat_store_dwordx2 v[52:53], v[0:1] offset:16
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 30, v12
+; CHECK-NEXT: v_lshrrev_b64 v[10:11], 1, v[10:11]
; CHECK-NEXT: v_or_b32_e32 v11, v11, v0
; CHECK-NEXT: flat_store_dwordx4 v[52:53], v[8:11]
; CHECK-NEXT: .LBB0_43: ; %Flow48
@@ -1122,21 +1121,22 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_cbranch_execz .LBB0_47
; CHECK-NEXT: ; %bb.46: ; %bb1b
; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
; CHECK-NEXT: v_lshrrev_b32_e32 v33, 15, v31
; CHECK-NEXT: v_alignbit_b32 v36, v31, v32, 15
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Reload
@@ -1159,10 +1159,10 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v32
; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v59
@@ -1186,8 +1186,8 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[8:9]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v10
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
; CHECK-NEXT: v_accvgpr_read_b32 v9, a33
@@ -1266,38 +1266,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_andn2_saveexec_b64 s[46:47], s[46:47]
; CHECK-NEXT: s_cbranch_execz .LBB0_49
; CHECK-NEXT: ; %bb.48: ; %bb1a
-; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword a0, off, s[0:3], s32 offset:1272 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword a1, off, s[0:3], s32 offset:1276 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword a2, off, s[0:3], s32 offset:1280 ; 4-byte Folded Reload
@@ -1330,58 +1298,84 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword a29, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:1136 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:1140 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a34, off, s[0:3], s32 offset:1144 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a35, off, s[0:3], s32 offset:1148 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a36, off, s[0:3], s32 offset:1152 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a37, off, s[0:3], s32 offset:1156 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a38, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a39, off, s[0:3], s32 offset:1164 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a40, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a41, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a42, off, s[0:3], s32 offset:1176 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a43, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a44, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a45, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a46, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a47, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a48, off, s[0:3], s32 offset:1200 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a49, off, s[0:3], s32 offset:1204 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a50, off, s[0:3], s32 offset:1208 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a51, off, s[0:3], s32 offset:1212 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a52, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a53, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a54, off, s[0:3], s32 offset:1224 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a55, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a56, off, s[0:3], s32 offset:1232 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a57, off, s[0:3], s32 offset:1236 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a58, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a59, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a60, off, s[0:3], s32 offset:1248 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a61, off, s[0:3], s32 offset:1252 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a62, off, s[0:3], s32 offset:1256 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword a63, off, s[0:3], s32 offset:1260 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a34
-; CHECK-NEXT: v_accvgpr_read_b32 v36, a38
-; CHECK-NEXT: v_accvgpr_read_b32 v37, a39
-; CHECK-NEXT: v_accvgpr_read_b32 v48, a50
-; CHECK-NEXT: v_accvgpr_read_b32 v59, a61
-; CHECK-NEXT: v_accvgpr_read_b32 v49, a51
-; CHECK-NEXT: v_accvgpr_read_b32 v53, a55
-; CHECK-NEXT: v_accvgpr_read_b32 v41, a43
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a36
-; CHECK-NEXT: v_accvgpr_read_b32 v38, a40
-; CHECK-NEXT: v_accvgpr_read_b32 v33, a35
-; CHECK-NEXT: v_accvgpr_read_b32 v39, a41
-; CHECK-NEXT: v_accvgpr_read_b32 v52, a54
-; CHECK-NEXT: v_accvgpr_read_b32 v40, a42
-; CHECK-NEXT: v_accvgpr_read_b32 v51, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v50, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v55, a57
-; CHECK-NEXT: v_accvgpr_read_b32 v43, a45
-; CHECK-NEXT: v_accvgpr_read_b32 v54, a56
-; CHECK-NEXT: v_accvgpr_read_b32 v42, a44
-; CHECK-NEXT: v_accvgpr_read_b32 v35, a37
-; CHECK-NEXT: v_accvgpr_read_b32 v44, a46
-; CHECK-NEXT: v_accvgpr_read_b32 v45, a47
-; CHECK-NEXT: v_accvgpr_read_b32 v46, a48
-; CHECK-NEXT: v_accvgpr_read_b32 v47, a49
-; CHECK-NEXT: v_accvgpr_read_b32 v56, a58
-; CHECK-NEXT: v_accvgpr_read_b32 v57, a59
-; CHECK-NEXT: v_accvgpr_read_b32 v58, a60
-; CHECK-NEXT: v_accvgpr_read_b32 v60, a62
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a28
-; CHECK-NEXT: v_accvgpr_read_b32 v61, a63
-; CHECK-NEXT: v_accvgpr_read_b32 v34, a30
; CHECK-NEXT: v_accvgpr_read_b32 v24, a24
; CHECK-NEXT: v_accvgpr_read_b32 v21, a21
; CHECK-NEXT: v_accvgpr_read_b32 v28, a28
; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a60
; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a56
; CHECK-NEXT: v_accvgpr_read_b32 v17, a17
; CHECK-NEXT: v_accvgpr_read_b32 v25, a25
-; CHECK-NEXT: v_cndmask_b32_e64 v36, v28, v32, s[44:45]
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
-; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
-; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
; CHECK-NEXT: v_accvgpr_read_b32 v29, a29
; CHECK-NEXT: v_accvgpr_read_b32 v25, a21
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a9
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a5
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v20, a20
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
+; CHECK-NEXT: v_accvgpr_read_b32 v13, a45
+; CHECK-NEXT: v_accvgpr_read_b32 v9, a41
+; CHECK-NEXT: v_accvgpr_read_b32 v5, a37
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
+; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
+; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v16, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v23, a23
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a27
+; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
+; CHECK-NEXT: v_accvgpr_read_b32 v22, a22
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a56
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a60
+; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a20
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a34
+; CHECK-NEXT: v_accvgpr_read_b32 v36, a38
+; CHECK-NEXT: v_accvgpr_read_b32 v32, a28
+; CHECK-NEXT: v_cndmask_b32_e64 v36, v28, v32, s[44:45]
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
+; CHECK-NEXT: v_accvgpr_read_b32 v37, a39
+; CHECK-NEXT: v_accvgpr_read_b32 v48, a50
+; CHECK-NEXT: v_accvgpr_read_b32 v59, a61
; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
; CHECK-NEXT: v_accvgpr_read_b32 v28, a24
; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
; CHECK-NEXT: v_accvgpr_read_b32 v24, a56
+; CHECK-NEXT: v_accvgpr_read_b32 v49, a51
; CHECK-NEXT: v_cndmask_b32_e64 v37, v59, v29, s[44:45]
; CHECK-NEXT: v_accvgpr_read_b32 v29, a25
; CHECK-NEXT: v_accvgpr_read_b32 v25, a57
@@ -1389,73 +1383,59 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
; CHECK-NEXT: v_cndmask_b32_e64 v48, v24, v28, s[42:43]
; CHECK-NEXT: v_accvgpr_read_b32 v28, a20
+; CHECK-NEXT: v_accvgpr_read_b32 v53, a55
; CHECK-NEXT: v_cndmask_b32_e64 v49, v25, v29, s[42:43]
; CHECK-NEXT: v_accvgpr_read_b32 v25, a21
; CHECK-NEXT: v_accvgpr_read_b32 v29, a21
; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
; CHECK-NEXT: v_accvgpr_read_b32 v21, a53
; CHECK-NEXT: v_accvgpr_read_b32 v28, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v13, a13
+; CHECK-NEXT: v_accvgpr_read_b32 v41, a43
; CHECK-NEXT: v_accvgpr_read_b32 v25, a17
; CHECK-NEXT: v_cndmask_b32_e64 v53, v21, v29, s[28:29]
; CHECK-NEXT: v_accvgpr_read_b32 v21, a17
; CHECK-NEXT: v_accvgpr_read_b32 v17, a49
; CHECK-NEXT: v_accvgpr_read_b32 v29, a17
; CHECK-NEXT: v_accvgpr_read_b32 v28, a12
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a9
; CHECK-NEXT: v_cndmask_b32_e64 v41, v17, v29, s[24:25]
-; CHECK-NEXT: v_accvgpr_read_b32 v13, a45
; CHECK-NEXT: v_accvgpr_read_b32 v29, a13
; CHECK-NEXT: v_accvgpr_read_b32 v25, a9
; CHECK-NEXT: v_accvgpr_read_b32 v21, a5
; CHECK-NEXT: v_accvgpr_read_b32 v17, a1
; CHECK-NEXT: v_accvgpr_read_b32 v28, a8
-; CHECK-NEXT: v_accvgpr_read_b32 v9, a41
; CHECK-NEXT: v_cndmask_b32_e64 v17, v13, v29, s[20:21]
; CHECK-NEXT: v_accvgpr_read_b32 v29, a9
; CHECK-NEXT: v_accvgpr_read_b32 v25, a5
; CHECK-NEXT: v_accvgpr_read_b32 v21, a1
; CHECK-NEXT: v_accvgpr_read_b32 v28, a6
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a5
; CHECK-NEXT: v_cndmask_b32_e64 v21, v9, v29, s[16:17]
; CHECK-NEXT: v_accvgpr_read_b32 v29, a7
; CHECK-NEXT: v_accvgpr_read_b32 v25, a3
; CHECK-NEXT: v_accvgpr_read_b32 v28, a4
-; CHECK-NEXT: v_accvgpr_read_b32 v5, a37
; CHECK-NEXT: v_accvgpr_read_b32 v29, a5
; CHECK-NEXT: v_accvgpr_read_b32 v25, a1
; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
; CHECK-NEXT: v_cndmask_b32_e64 v25, v5, v29, s[12:13]
; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a1
; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a32
-; CHECK-NEXT: v_accvgpr_read_b32 v1, a33
; CHECK-NEXT: v_accvgpr_read_b32 v29, a1
; CHECK-NEXT: v_accvgpr_read_b32 v28, a0
; CHECK-NEXT: v_cndmask_b32_e64 v29, v1, v29, s[4:5]
; CHECK-NEXT: v_cndmask_b32_e64 v28, v0, v28, s[4:5]
; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a20
-; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
-; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
; CHECK-NEXT: v_accvgpr_read_b32 v30, a32
; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v16, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a23
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a36
; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a27
+; CHECK-NEXT: v_accvgpr_read_b32 v38, a40
+; CHECK-NEXT: v_accvgpr_read_b32 v34, a30
; CHECK-NEXT: v_accvgpr_read_b32 v30, a62
-; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
; CHECK-NEXT: v_accvgpr_read_b32 v20, a52
-; CHECK-NEXT: v_accvgpr_read_b32 v22, a22
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
; CHECK-NEXT: v_cndmask_b32_e64 v38, v30, v34, s[40:41]
; CHECK-NEXT: v_accvgpr_read_b32 v30, a26
; CHECK-NEXT: v_accvgpr_read_b32 v27, a23
@@ -1465,6 +1445,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v19, a19
; CHECK-NEXT: v_accvgpr_read_b32 v26, a26
; CHECK-NEXT: v_accvgpr_read_b32 v31, a33
+; CHECK-NEXT: v_accvgpr_read_b32 v33, a35
; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
; CHECK-NEXT: v_accvgpr_read_b32 v27, a59
; CHECK-NEXT: v_accvgpr_read_b32 v23, a19
@@ -1472,6 +1453,8 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v24, a16
; CHECK-NEXT: v_accvgpr_read_b32 v16, a48
; CHECK-NEXT: v_accvgpr_read_b32 v20, a16
+; CHECK-NEXT: v_accvgpr_read_b32 v39, a41
+; CHECK-NEXT: v_accvgpr_read_b32 v52, a54
; CHECK-NEXT: v_accvgpr_read_b32 v33, a31
; CHECK-NEXT: v_accvgpr_read_b32 v31, a63
; CHECK-NEXT: v_accvgpr_read_b32 v26, a22
@@ -1485,6 +1468,8 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v30, a18
; CHECK-NEXT: v_accvgpr_read_b32 v12, a12
; CHECK-NEXT: v_accvgpr_read_b32 v18, a18
+; CHECK-NEXT: v_accvgpr_read_b32 v40, a42
+; CHECK-NEXT: v_accvgpr_read_b32 v51, a53
; CHECK-NEXT: v_cndmask_b32_e64 v39, v31, v33, s[40:41]
; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
; CHECK-NEXT: v_accvgpr_read_b32 v22, a18
@@ -1509,6 +1494,8 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v16, a0
; CHECK-NEXT: v_accvgpr_read_b32 v30, a12
; CHECK-NEXT: v_accvgpr_read_b32 v8, a8
+; CHECK-NEXT: v_accvgpr_read_b32 v50, a52
+; CHECK-NEXT: v_accvgpr_read_b32 v55, a57
; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
; CHECK-NEXT: v_accvgpr_read_b32 v26, a58
; CHECK-NEXT: v_accvgpr_read_b32 v34, a26
@@ -1523,6 +1510,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_cndmask_b32_e64 v16, v12, v30, s[20:21]
; CHECK-NEXT: v_accvgpr_read_b32 v30, a10
; CHECK-NEXT: v_accvgpr_read_b32 v15, a15
+; CHECK-NEXT: v_accvgpr_read_b32 v43, a45
; CHECK-NEXT: v_cndmask_b32_e64 v50, v26, v34, s[26:27]
; CHECK-NEXT: v_accvgpr_read_b32 v26, a18
; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
@@ -1539,6 +1527,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v30, a8
; CHECK-NEXT: v_accvgpr_read_b32 v4, a4
; CHECK-NEXT: v_accvgpr_read_b32 v11, a11
+; CHECK-NEXT: v_accvgpr_read_b32 v54, a56
; CHECK-NEXT: v_accvgpr_read_b32 v18, a50
; CHECK-NEXT: v_accvgpr_read_b32 v22, a54
; CHECK-NEXT: v_accvgpr_read_b32 v34, a22
@@ -1555,6 +1544,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_accvgpr_read_b32 v30, a6
; CHECK-NEXT: v_accvgpr_read_b32 v7, a7
; CHECK-NEXT: v_accvgpr_read_b32 v14, a14
+; CHECK-NEXT: v_accvgpr_read_b32 v42, a44
; CHECK-NEXT: v_cndmask_b32_e64 v54, v22, v34, s[22:23]
; CHECK-NEXT: v_accvgpr_read_b32 v4, a36
; CHECK-NEXT: v_accvgpr_read_b32 v11, a43
@@ -1619,6 +1609,16 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, 0x70, v0
; CHECK-NEXT: v_accvgpr_read_b32 v32, a6
; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; CHECK-NEXT: v_accvgpr_read_b32 v35, a37
+; CHECK-NEXT: v_accvgpr_read_b32 v44, a46
+; CHECK-NEXT: v_accvgpr_read_b32 v45, a47
+; CHECK-NEXT: v_accvgpr_read_b32 v46, a48
+; CHECK-NEXT: v_accvgpr_read_b32 v47, a49
+; CHECK-NEXT: v_accvgpr_read_b32 v56, a58
+; CHECK-NEXT: v_accvgpr_read_b32 v57, a59
+; CHECK-NEXT: v_accvgpr_read_b32 v58, a60
+; CHECK-NEXT: v_accvgpr_read_b32 v60, a62
+; CHECK-NEXT: v_accvgpr_read_b32 v61, a63
; CHECK-NEXT: v_accvgpr_read_b32 v32, a2
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[16:19]
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[40:43] offset:64
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
index e95ba43686e3a..61f46d2cd7b27 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
@@ -2167,8 +2167,8 @@ define double @test_vector_reduce_fadd_v16double(double %sp, <16 x double> %v) {
; GFX9-SDAG-LABEL: test_vector_reduce_fadd_v16double:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: scratch_load_dword v33, off, s32 offset:8
; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
+; GFX9-SDAG-NEXT: scratch_load_dword v33, off, s32 offset:8
; GFX9-SDAG-NEXT: scratch_load_dword v32, off, s32 offset:4
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[4:5]
@@ -2184,7 +2184,7 @@ define double @test_vector_reduce_fadd_v16double(double %sp, <16 x double> %v) {
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[24:25]
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[26:27]
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[28:29]
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[30:31]
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[32:33]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
index 6291173230114..40732a69761ac 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
@@ -2167,8 +2167,8 @@ define double @test_vector_reduce_fmul_v16double(double %sp, <16 x double> %v) {
; GFX9-SDAG-LABEL: test_vector_reduce_fmul_v16double:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: scratch_load_dword v33, off, s32 offset:8
; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
+; GFX9-SDAG-NEXT: scratch_load_dword v33, off, s32 offset:8
; GFX9-SDAG-NEXT: scratch_load_dword v32, off, s32 offset:4
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[2:3]
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[4:5]
@@ -2184,7 +2184,7 @@ define double @test_vector_reduce_fmul_v16double(double %sp, <16 x double> %v) {
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[24:25]
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[26:27]
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[28:29]
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(1)
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(2)
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[30:31]
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: v_mul_f64 v[0:1], v[0:1], v[32:33]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
index 735b8e0f74089..991e11f23b2aa 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
@@ -397,23 +397,23 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: global_load_b128 v[56:59], v[0:1], off
; CHECK-NEXT: global_load_b128 v[104:107], v[2:3], off
; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, 0
-; CHECK-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0x70
-; CHECK-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, 0x80
+; CHECK-NEXT: v_dual_mov_b32 v2, 0x70 :: v_dual_mov_b32 v9, 0
+; CHECK-NEXT: v_dual_mov_b32 v8, 0x80 :: v_dual_mov_b32 v1, 0
; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: global_load_b128 v[108:111], v[4:5], off
; CHECK-NEXT: global_load_b128 v[60:63], v[6:7], off
+; CHECK-NEXT: global_load_b128 v[108:111], v[4:5], off
; CHECK-NEXT: v_mov_b32_e32 v5, 0
; CHECK-NEXT: s_clause 0x4
-; CHECK-NEXT: global_load_b128 v[72:75], v[0:1], off
; CHECK-NEXT: global_load_b128 v[10:13], v[2:3], off
; CHECK-NEXT: global_load_b128 v[14:17], v[8:9], off
; CHECK-NEXT: global_load_b128 v[18:21], v[8:9], off offset:16
; CHECK-NEXT: global_load_b128 v[22:25], v[8:9], off offset:32
+; CHECK-NEXT: global_load_b128 v[72:75], v[0:1], off
; CHECK-NEXT: v_dual_mov_b32 v4, 32 :: v_dual_mov_b32 v7, 0
; CHECK-NEXT: v_mov_b32_e32 v6, 16
; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: global_load_b128 v[76:79], v[4:5], off
; CHECK-NEXT: global_load_b128 v[88:91], v[6:7], off
+; CHECK-NEXT: global_load_b128 v[76:79], v[4:5], off
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_sext_i32_i16 s1, s1
@@ -429,13 +429,13 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b64 s[36:37], s[8:9]
; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
; CHECK-NEXT: s_mov_b64 s[48:49], s[4:5]
-; CHECK-NEXT: s_wait_loadcnt 0x5
+; CHECK-NEXT: s_wait_loadcnt 0x6
; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:148 ; 16-byte Folded Spill
-; CHECK-NEXT: s_wait_loadcnt 0x4
+; CHECK-NEXT: s_wait_loadcnt 0x5
; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:164 ; 16-byte Folded Spill
-; CHECK-NEXT: s_wait_loadcnt 0x3
+; CHECK-NEXT: s_wait_loadcnt 0x4
; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:180 ; 16-byte Folded Spill
-; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: s_wait_loadcnt 0x3
; CHECK-NEXT: s_clause 0x4 ; 80-byte Folded Spill
; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:196
; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:212
@@ -466,14 +466,14 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
; CHECK-NEXT: s_clause 0x7 ; 128-byte Folded Reload
-; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:276 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:292 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:308 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:324 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:340 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:356 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:372 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:388 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:276 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:324 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:308 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:292 th:TH_LOAD_LU
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_sext_i32_i16 s1, s1
@@ -489,23 +489,26 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b32 s13, s52
; CHECK-NEXT: s_mov_b32 s14, s51
; CHECK-NEXT: s_mov_b32 s15, s50
-; CHECK-NEXT: s_wait_loadcnt 0x3
+; CHECK-NEXT: s_wait_loadcnt 0x7
; CHECK-NEXT: scratch_store_b32 off, v19, s32 offset:128
-; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: s_wait_loadcnt 0x6
; CHECK-NEXT: scratch_load_b128 v[19:22], off, s33 offset:148 th:TH_LOAD_LU ; 16-byte Folded Reload
-; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: s_wait_loadcnt 0x6
; CHECK-NEXT: scratch_load_b128 v[23:26], off, s33 offset:164 th:TH_LOAD_LU ; 16-byte Folded Reload
-; CHECK-NEXT: s_wait_loadcnt 0x2
+; CHECK-NEXT: s_wait_loadcnt 0x6
; CHECK-NEXT: s_clause 0x5 ; 96-byte Folded Reload
; CHECK-NEXT: scratch_load_b128 v[27:30], off, s33 offset:180 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[31:34], off, s33 offset:196 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[35:38], off, s33 offset:212 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[39:42], off, s33 offset:228 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[43:46], off, s33 offset:244 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[39:42], off, s33 offset:228 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[35:38], off, s33 offset:212 th:TH_LOAD_LU
; CHECK-NEXT: scratch_load_b128 v[47:50], off, s33 offset:260 th:TH_LOAD_LU
-; CHECK-NEXT: s_clause 0x3
+; CHECK-NEXT: s_wait_loadcnt 0xa
; CHECK-NEXT: scratch_store_b128 off, v[15:18], s32 offset:112
+; CHECK-NEXT: s_wait_loadcnt 0x9
; CHECK-NEXT: scratch_store_b128 off, v[11:14], s32 offset:96
+; CHECK-NEXT: s_wait_loadcnt 0x8
+; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: scratch_store_b128 off, v[7:10], s32 offset:80
; CHECK-NEXT: scratch_store_b128 off, v[3:6], s32 offset:64
; CHECK-NEXT: v_dual_mov_b32 v4, v88 :: v_dual_mov_b32 v5, v89
@@ -516,13 +519,15 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: v_dual_mov_b32 v14, v62 :: v_dual_mov_b32 v15, v63
; CHECK-NEXT: v_dual_mov_b32 v16, v108 :: v_dual_mov_b32 v17, v109
; CHECK-NEXT: v_mov_b32_e32 v18, v110
-; CHECK-NEXT: s_wait_loadcnt 0x1
+; CHECK-NEXT: s_wait_loadcnt 0x3
; CHECK-NEXT: v_dual_mov_b32 v44, v0 :: v_dual_mov_b32 v45, v1
; CHECK-NEXT: v_mov_b32_e32 v46, v2
; CHECK-NEXT: v_dual_mov_b32 v0, v72 :: v_dual_mov_b32 v1, v73
; CHECK-NEXT: v_dual_mov_b32 v2, v74 :: v_dual_mov_b32 v3, v75
+; CHECK-NEXT: s_wait_loadcnt 0x2
; CHECK-NEXT: v_dual_mov_b32 v43, v34 :: v_dual_mov_b32 v42, v33
; CHECK-NEXT: v_dual_mov_b32 v41, v32 :: v_dual_mov_b32 v40, v31
+; CHECK-NEXT: s_wait_loadcnt 0x1
; CHECK-NEXT: v_dual_mov_b32 v39, v30 :: v_dual_mov_b32 v38, v29
; CHECK-NEXT: v_dual_mov_b32 v37, v28 :: v_dual_mov_b32 v36, v27
; CHECK-NEXT: v_dual_mov_b32 v35, v26 :: v_dual_mov_b32 v34, v25
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall-call-target-av-register-failure.ll b/llvm/test/CodeGen/AMDGPU/waterfall-call-target-av-register-failure.ll
index 51303f11a8f47..79b61f02c4ea3 100644
--- a/llvm/test/CodeGen/AMDGPU/waterfall-call-target-av-register-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/waterfall-call-target-av-register-failure.ll
@@ -38,11 +38,12 @@ define i32 @fix_sgpr_copies_indirect_call(ptr addrspace(5) %ptr) #0 {
; CHECK-NEXT: s_or_saveexec_b64 s[34:35], -1
; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b64 exec, s[34:35]
-; CHECK-NEXT: buffer_load_dword v0, v1, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v2, v1, s[0:3], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v0, v1, s[0:3], 0 offen
; CHECK-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: v_mov_b32_e32 v1, v2
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
; CHECK-NEXT: s_nop 0
; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
index db1faadee76ca..178318a0ddc3d 100644
--- a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
@@ -384,8 +384,8 @@ define amdgpu_gfx_whole_wave i32 @csr(i1 %active, i32 %a, i32 %b) #0 {
; DAGISEL-NEXT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; DAGISEL-NEXT: s_xor_b32 exec_lo, vcc_lo, -1
; DAGISEL-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; DAGISEL-NEXT: scratch_load_b32 v2, off, s32
; DAGISEL-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; DAGISEL-NEXT: scratch_load_b32 v2, off, s32
; DAGISEL-NEXT: scratch_load_b32 v1, off, s32 offset:8
; DAGISEL-NEXT: scratch_load_b32 v49, off, s32 offset:16
; DAGISEL-NEXT: s_mov_b32 exec_lo, vcc_lo
@@ -423,8 +423,8 @@ define amdgpu_gfx_whole_wave i32 @csr(i1 %active, i32 %a, i32 %b) #0 {
; GISEL-NEXT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; GISEL-NEXT: s_xor_b32 exec_lo, vcc_lo, -1
; GISEL-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; GISEL-NEXT: scratch_load_b32 v2, off, s32
; GISEL-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GISEL-NEXT: scratch_load_b32 v2, off, s32
; GISEL-NEXT: scratch_load_b32 v1, off, s32 offset:8
; GISEL-NEXT: scratch_load_b32 v49, off, s32 offset:16
; GISEL-NEXT: s_mov_b32 exec_lo, vcc_lo
@@ -463,8 +463,8 @@ define amdgpu_gfx_whole_wave i32 @csr(i1 %active, i32 %a, i32 %b) #0 {
; DAGISEL64-NEXT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; DAGISEL64-NEXT: s_xor_b64 exec, vcc, -1
; DAGISEL64-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; DAGISEL64-NEXT: scratch_load_b32 v2, off, s32
; DAGISEL64-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; DAGISEL64-NEXT: scratch_load_b32 v2, off, s32
; DAGISEL64-NEXT: scratch_load_b32 v1, off, s32 offset:8
; DAGISEL64-NEXT: scratch_load_b32 v49, off, s32 offset:16
; DAGISEL64-NEXT: s_mov_b64 exec, vcc
@@ -503,8 +503,8 @@ define amdgpu_gfx_whole_wave i32 @csr(i1 %active, i32 %a, i32 %b) #0 {
; GISEL64-NEXT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; GISEL64-NEXT: s_xor_b64 exec, vcc, -1
; GISEL64-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; GISEL64-NEXT: scratch_load_b32 v2, off, s32
; GISEL64-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GISEL64-NEXT: scratch_load_b32 v2, off, s32
; GISEL64-NEXT: scratch_load_b32 v1, off, s32 offset:8
; GISEL64-NEXT: scratch_load_b32 v49, off, s32 offset:16
; GISEL64-NEXT: s_mov_b64 exec, vcc
@@ -541,8 +541,8 @@ define amdgpu_gfx_whole_wave i32 @csr(i1 %active, i32 %a, i32 %b) #0 {
; GFX1250-DAGISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-DAGISEL-NEXT: s_xor_b32 exec_lo, vcc_lo, -1
; GFX1250-DAGISEL-NEXT: s_clause 0x3 ; 16-byte Folded Reload
-; GFX1250-DAGISEL-NEXT: scratch_load_b32 v2, off, s32 nv
; GFX1250-DAGISEL-NEXT: scratch_load_b32 v0, off, s32 offset:4 nv
+; GFX1250-DAGISEL-NEXT: scratch_load_b32 v2, off, s32 nv
; GFX1250-DAGISEL-NEXT: scratch_load_b32 v1, off, s32 offset:8 nv
; GFX1250-DAGISEL-NEXT: scratch_load_b32 v49, off, s32 offset:16 nv
; GFX1250-DAGISEL-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
index 9989907f4b9b9..f5870f3b07d6e 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
@@ -188,9 +188,9 @@ define amdgpu_gfx void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg)
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_2
; GFX9-O0-NEXT: ; %bb.1: ; %if
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
@@ -958,34 +958,37 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v26, v38
; GFX9-O0-NEXT: v_mov_b32_e32 v27, v37
; GFX9-O0-NEXT: v_mov_b32_e32 v28, v36
-; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-O0-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(15)
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(14)
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9
; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(13)
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v11
; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(12)
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v11
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
@@ -995,6 +998,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v32, s37
; GFX9-O0-NEXT: v_cndmask_b32_e64 v32, v32, v0, s[34:35]
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(11)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v10
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: s_mov_b32 s36, s38
@@ -1011,6 +1015,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v32, s37
; GFX9-O0-NEXT: v_cndmask_b32_e64 v32, v32, v0, s[34:35]
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v8
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: ; implicit-def: $sgpr38_sgpr39
@@ -1026,6 +1031,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v32, s37
; GFX9-O0-NEXT: v_cndmask_b32_e64 v32, v32, v0, s[34:35]
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: ; implicit-def: $sgpr38_sgpr39
@@ -1041,6 +1047,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v32, s37
; GFX9-O0-NEXT: v_cndmask_b32_e64 v32, v32, v0, s[34:35]
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: ; implicit-def: $sgpr38_sgpr39
@@ -1056,6 +1063,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v32, s37
; GFX9-O0-NEXT: v_cndmask_b32_e64 v32, v32, v0, s[34:35]
; GFX9-O0-NEXT: s_mov_b64 exec, s[34:35]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(7)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-O0-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O0-NEXT: ; implicit-def: $sgpr38_sgpr39
@@ -1066,6 +1074,7 @@ define amdgpu_gfx <32 x i32> @strict_wwm_callee_saves(<32 x i32> inreg %keep, pt
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v33
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v34
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v11
+; GFX9-O0-NEXT: s_waitcnt vmcnt(6)
; GFX9-O0-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v10
; GFX9-O0-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 0ef5725607920..916e17753a166 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -165,9 +165,9 @@ define amdgpu_cs void @cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_2
; GFX9-O0-NEXT: ; %bb.1: ; %if
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-O0-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
@@ -1028,9 +1028,9 @@ define amdgpu_cs void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-O0-NEXT: s_cbranch_execz .LBB8_2
; GFX9-O0-NEXT: ; %bb.1: ; %if
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4
; GFX9-O0-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
More information about the llvm-commits
mailing list