[llvm-branch-commits] [llvm] [InlineSpiller][AMDGPU] Implement subreg reload during RA spill (PR #175002)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 31 02:21:02 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/175002
>From 1a848f4a7ae6bcb78e5320aebe55f879124b9e95 Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Thu, 8 Jan 2026 07:53:18 +0000
Subject: [PATCH 1/2] [InlineSpiller][AMDGPU] Implement subreg reload during RA
spill
Currently, when a virtual register is partially used, the
entire tuple is restored from the spilled location, even if
only a subset of its sub-registers is needed. This patch
introduces support for partial reloads by analyzing actual
register usage and restoring only the required sub-registers.
This improvement enhances register allocation efficiency,
particularly for cases involving tuple virtual registers.
For AMDGPU, this change brings considerable improvements
in workloads that involve matrix operations, large vectors,
and complex control flows.
---
.../include/llvm/CodeGen/TargetRegisterInfo.h | 10 +
llvm/lib/CodeGen/InlineSpiller.cpp | 57 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 24 +-
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 14 +
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 5 +
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 10724 +++++++---------
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 156 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll | 29 +-
.../dummy-regalloc-priority-advisor.mir | 4 +-
.../AMDGPU/gfx-callable-return-types.ll | 21 +-
.../AMDGPU/hsa-metadata-kernel-code-props.ll | 4 +-
.../identical-subrange-spill-infloop.ll | 91 +-
...-reg-class-snippet-copy-use-after-free.mir | 1 +
...nfloop-subrange-spill-inspect-subrange.mir | 5 +-
.../CodeGen/AMDGPU/infloop-subrange-spill.mir | 2 +-
.../ra-inserted-scalar-instructions.mir | 76 +-
.../ran-out-of-sgprs-allocation-failure.mir | 187 +-
...pressure-mitigation-with-subreg-reload.mir | 50 +-
...kip-partial-reload-for-16bit-regaccess.mir | 91 +
.../CodeGen/AMDGPU/spill-scavenge-offset.ll | 34 +-
.../CodeGen/AMDGPU/splitkit-copy-bundle.mir | 14 +-
.../AMDGPU/splitkit-copy-live-lanes.mir | 72 +-
.../splitkit-getsubrangeformask-phi-extend.ll | 85 +-
.../AMDGPU/splitkit-nolivesubranges.mir | 4 +-
...-last-chance-recoloring-alloc-succeeds.mir | 22 +-
...se-after-free-after-cleanup-failed-vreg.ll | 16 -
.../AMDGPU/vgpr-mark-last-scratch-load.ll | 190 +-
27 files changed, 5652 insertions(+), 6336 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
delete mode 100644 llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 3446abcff5875..38ffe35010379 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1060,6 +1060,11 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
return true;
}
+ /// To enable subreg reload of register tuples during RA. This would
+ /// eventually improve the register allocation for the functions that involve
+ /// subreg uses instead of the entire tuple.
+ virtual bool shouldEnableSubRegReload(unsigned SubReg) const { return false; }
+
/// When prioritizing live ranges in register allocation, if this hook returns
/// true then the AllocationPriority of the register class will be treated as
/// more important than whether the range is local to a basic block or global.
@@ -1083,6 +1088,11 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
bool checkAllSuperRegsMarked(const BitVector &RegisterSet,
ArrayRef<MCPhysReg> Exceptions = ArrayRef<MCPhysReg>()) const;
+ virtual const TargetRegisterClass *
+ getConstrainedRegClass(const TargetRegisterClass *RC) const {
+ return RC;
+ }
+
virtual const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index f3682a2e24808..6deab6fefb2a3 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -225,7 +225,8 @@ class InlineSpiller : public Spiller {
bool coalesceStackAccess(MachineInstr *MI, Register Reg);
bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>,
MachineInstr *LoadMI = nullptr);
- void insertReload(Register VReg, SlotIndex, MachineBasicBlock::iterator MI);
+ void insertReload(Register VReg, unsigned SubReg, SlotIndex,
+ MachineBasicBlock::iterator MI);
void insertSpill(Register VReg, bool isKill, MachineBasicBlock::iterator MI);
void spillAroundUses(Register Reg);
@@ -1244,14 +1245,14 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
return true;
}
-void InlineSpiller::insertReload(Register NewVReg,
+void InlineSpiller::insertReload(Register NewVReg, unsigned SubReg,
SlotIndex Idx,
MachineBasicBlock::iterator MI) {
MachineBasicBlock &MBB = *MI->getParent();
MachineInstrSpan MIS(MI, &MBB);
TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot,
- MRI.getRegClass(NewVReg), Register());
+ MRI.getRegClass(NewVReg), Register(), SubReg);
LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MI);
@@ -1380,10 +1381,51 @@ void InlineSpiller::spillAroundUses(Register Reg) {
// Create a new virtual register for spill/fill.
// FIXME: Infer regclass from instruction alone.
- Register NewVReg = Edit->createFrom(Reg);
+
+ unsigned SubReg = 0;
+ LaneBitmask CoveringLanes = LaneBitmask::getNone();
+ // If the subreg liveness is enabled, identify the subreg use(s) to try
+ // subreg reload. Skip if the instruction also defines the register.
+ // For copy bundles, get the covering lane masks.
+ if (MRI.subRegLivenessEnabled() && !RI.Writes) {
+ for (auto [MI, OpIdx] : Ops) {
+ const MachineOperand &MO = MI->getOperand(OpIdx);
+ assert(MO.isReg() && MO.getReg() == Reg);
+ if (MO.isUse()) {
+ SubReg = MO.getSubReg();
+ if (SubReg)
+ CoveringLanes |= TRI.getSubRegIndexLaneMask(SubReg);
+ }
+ }
+ }
+
+ if (MI.isBundled() && CoveringLanes.any()) {
+ CoveringLanes = LaneBitmask(bit_ceil(CoveringLanes.getAsInteger()) - 1);
+ // Obtain the covering subregister index, including any missing indices
+ // within the identified small range. Although this may be suboptimal due
+ // to gaps in the subregisters that are not part of the copy bundle, it is
+ // benificial when components outside this range of the original tuple can
+ // be completely skipped from the reload.
+ SubReg = TRI.getSubRegIdxFromLaneMask(CoveringLanes);
+ }
+
+ // If the target doesn't support subreg reload, fallback to restoring the
+ // full tuple.
+ if (SubReg && !TRI.shouldEnableSubRegReload(SubReg))
+ SubReg = 0;
+
+ const TargetRegisterClass *OrigRC = MRI.getRegClass(Reg);
+ const TargetRegisterClass *NewRC =
+ SubReg ? TRI.getSubRegisterClass(OrigRC, SubReg) : nullptr;
+
+ // Check if the target needs to constrain the RC further.
+ if (NewRC)
+ NewRC = TRI.getConstrainedRegClass(NewRC);
+
+ Register NewVReg = Edit->createFrom(Reg, NewRC);
if (RI.Reads)
- insertReload(NewVReg, Idx, &MI);
+ insertReload(NewVReg, SubReg, Idx, &MI);
// Rewrite instruction operands.
bool hasLiveDef = false;
@@ -1391,7 +1433,10 @@ void InlineSpiller::spillAroundUses(Register Reg) {
MachineOperand &MO = OpPair.first->getOperand(OpPair.second);
MO.setReg(NewVReg);
if (MO.isUse()) {
- if (!OpPair.first->isRegTiedToDefOperand(OpPair.second))
+ if (SubReg && !MI.isBundled())
+ MO.setSubReg(0);
+ if (!OpPair.first->isRegTiedToDefOperand(OpPair.second) ||
+ (SubReg && !MI.isBundled()))
MO.setIsKill();
} else {
if (!MO.isDead())
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c95f1703d4a27..c294de7ad1004 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1817,12 +1817,23 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
const DebugLoc &DL = MBB.findDebugLoc(MI);
unsigned SpillSize = RI.getSpillSize(*RC);
+ unsigned SubRegIdx = 0;
+ if (SubReg) {
+ uint64_t Mask = RI.getSubRegIndexLaneMask(SubReg).getAsInteger();
+ assert(llvm::popcount(Mask) % 2 == 0 &&
+ "expected only 32-bit subreg access");
+
+ // For subreg reload, identify the start offset. Each 32-bit register
+ // consists of two regunits and eventually two bits in the Lanemask.
+ SubRegIdx = llvm::countr_zero(Mask) / 2;
+ }
+
MachinePointerInfo PtrInfo
= MachinePointerInfo::getFixedStack(*MF, FrameIndex);
- MachineMemOperand *MMO = MF->getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
- FrameInfo.getObjectAlign(FrameIndex));
+ MachineMemOperand *MMO =
+ MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, SpillSize,
+ FrameInfo.getObjectAlign(FrameIndex));
if (RI.isSGPRClass(RC)) {
if (FrameInfo.getStackID(FrameIndex) == TargetStackID::SGPRSpill)
@@ -1841,19 +1852,22 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
BuildMI(MBB, MI, DL, OpDesc, DestReg)
.addFrameIndex(FrameIndex) // addr
- .addImm(0) // offset
+ .addImm(SubRegIdx) // offset
.addMemOperand(MMO)
.addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
return;
}
+ // Convert the subreg index to stack offset.
+ SubRegIdx *= 4;
+
unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC,
SpillSize, *MFI);
BuildMI(MBB, MI, DL, get(Opcode), DestReg)
.addFrameIndex(FrameIndex) // vaddr
.addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
- .addImm(0) // offset
+ .addImm(SubRegIdx) // offset
.addMemOperand(MMO);
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 4f7901f316732..c8974cbf4885b 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3868,6 +3868,15 @@ bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI,
return RC && isAGPRClass(RC);
}
+bool SIRegisterInfo::shouldEnableSubRegReload(unsigned SubReg) const {
+ // Disable lo16 and hi16 (16-bit) accesses as they are subreg views of the
+ // same 32-bit register and don't represent independent storage. If the number
+ // of bits set in the mask is odd, it indicates the presence of a 16-bit
+ // access as each 32-bit register consists of two Regunits and they take two
+ // bits in the regmask.
+ return getSubRegIndexLaneMask(SubReg).getNumLanes() % 2 == 0;
+}
+
unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
MachineFunction &MF) const {
unsigned MinOcc = ST.getOccupancyWithWorkGroupSizes(MF).first;
@@ -4001,6 +4010,11 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
}
}
+const TargetRegisterClass *
+SIRegisterInfo::getConstrainedRegClass(const TargetRegisterClass *RC) const {
+ return getProperlyAlignedRC(RC);
+}
+
const TargetRegisterClass *
SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index e464c9334ffea..1ae4594632397 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -353,6 +353,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
ArrayRef<int16_t> getRegSplitParts(const TargetRegisterClass *RC,
unsigned EltSize) const;
+ bool shouldEnableSubRegReload(unsigned SubReg) const override;
+
unsigned getRegPressureLimit(const TargetRegisterClass *RC,
MachineFunction &MF) const override;
@@ -376,6 +378,9 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
return getRegClassForSizeOnBank(Ty.getSizeInBits(), Bank);
}
+ const TargetRegisterClass *
+ getConstrainedRegClass(const TargetRegisterClass *RC) const override;
+
const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 3e0fd263fcc5c..08768d88da028 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -5832,60 +5832,55 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: .LBB12_4: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
+; VI-NEXT: v_perm_b32 v13, v13, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v7, v7, v38, s4
+; VI-NEXT: v_perm_b32 v11, v11, v48, s4
; VI-NEXT: v_perm_b32 v10, v10, v37, s4
; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v13, v13, v35, s4
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v8, v8, v33, s4
; VI-NEXT: v_perm_b32 v33, v49, v53, s4
-; VI-NEXT: v_perm_b32 v11, v11, v48, s4
-; VI-NEXT: v_perm_b32 v3, v3, v34, s4
-; VI-NEXT: v_perm_b32 v34, v47, v39, s4
-; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v55, v59, v41, s4
; VI-NEXT: v_perm_b32 v1, v1, v52, s4
; VI-NEXT: v_perm_b32 v2, v2, v58, s4
; VI-NEXT: v_perm_b32 v51, v51, v40, s4
+; VI-NEXT: v_perm_b32 v3, v3, v34, s4
; VI-NEXT: v_perm_b32 v4, v4, v57, s4
+; VI-NEXT: v_perm_b32 v34, v47, v39, s4
; VI-NEXT: v_perm_b32 v5, v5, v56, s4
; VI-NEXT: v_perm_b32 v6, v6, v46, s4
+; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v9, v9, v45, s4
; VI-NEXT: v_perm_b32 v12, v12, v63, s4
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v36, v36, v37, s4
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v14, v14, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v35, v38, v35, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v15, v15, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v14, v14, v35, s4
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v35, v35, v48, s4
+; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v38, v38, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v36, v36, v37, s4
+; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v17, v17, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v18, v18, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v48, v48, v49, s4
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v55
; VI-NEXT: v_or_b32_e32 v1, v1, v49
@@ -5987,12 +5982,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v21, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6007,12 +6002,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6027,12 +6022,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6047,12 +6042,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6067,12 +6062,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v29, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6087,12 +6082,12 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v31, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -6750,61 +6745,55 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v11, v11, v36, s4
; GFX9-NEXT: v_perm_b32 v12, v12, v35, s4
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v34, v38, v39, s4
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v33, s4
; GFX9-NEXT: v_perm_b32 v33, v50, v54, s4
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v55, v58, v41, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v59, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
; GFX9-NEXT: v_perm_b32 v2, v2, v52, s4
-; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
+; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v52, v57, v40, s4
; GFX9-NEXT: v_perm_b32 v5, v5, v56, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v47, s4
; GFX9-NEXT: v_perm_b32 v8, v8, v46, s4
+; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v45, s4
; GFX9-NEXT: v_perm_b32 v10, v10, v44, s4
; GFX9-NEXT: v_perm_b32 v13, v13, v60, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v35, v36, v35, s4
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v39, v36, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v15, v15, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
+; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v39, v39, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v17, v17, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v50, v50, v53, s4
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v50, v50, v51, s4
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GFX9-NEXT: v_perm_b32 v1, v49, v43, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
@@ -6894,10 +6883,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v21, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -6915,10 +6903,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -6936,10 +6923,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -6957,10 +6943,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
@@ -6978,10 +6963,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
@@ -6999,10 +6983,9 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -8777,20 +8760,18 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_and_b32 s17, s63, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s93, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 0
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 1
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 2
+; SI-NEXT: v_readlane_b32 s17, v22, 0
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s22, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 3
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 2
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 4
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8804,23 +8785,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s50, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s81, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 5
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s80, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 6
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 7
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 8
+; SI-NEXT: v_readlane_b32 s17, v22, 6
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s24, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 9
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 8
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 10
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8834,22 +8812,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s71, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s69, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 11
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s59, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 12
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s19, v22, 13
+; SI-NEXT: v_readlane_b32 s17, v22, 12
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s26, 0xff
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 14
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 14
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s68, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -8862,23 +8838,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s79, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s67, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 15
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s39, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 16
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 17
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 18
+; SI-NEXT: v_readlane_b32 s17, v22, 16
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s28, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 19
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 18
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 20
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8892,23 +8865,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s49, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s48, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 21
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s38, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 22
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 23
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 24
+; SI-NEXT: v_readlane_b32 s17, v22, 22
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 25
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 24
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 26
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8922,23 +8892,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s66, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s51, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 27
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s46, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 28
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 29
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 30
+; SI-NEXT: v_readlane_b32 s17, v22, 28
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 31
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 30
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 32
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8952,23 +8919,20 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s37, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s56, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 33
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s99, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 34
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 35
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 36
+; SI-NEXT: v_readlane_b32 s17, v22, 34
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 37
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 36
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 38
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -8992,16 +8956,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_readlane_b32 s16, v22, 40
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v22, 41
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 39
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v22, 42
-; SI-NEXT: v_readlane_b32 s17, v22, 43
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 44
+; SI-NEXT: v_readlane_b32 s17, v22, 44
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_or_b32 s14, s14, s16
@@ -9024,15 +8985,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_readlane_b32 s14, v22, 46
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 47
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v22, 48
-; SI-NEXT: v_readlane_b32 s15, v22, 49
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 50
+; SI-NEXT: v_readlane_b32 s15, v22, 50
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -9054,15 +9013,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s12
; SI-NEXT: v_readlane_b32 s12, v22, 52
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v22, 53
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: v_readlane_b32 s12, v22, 54
-; SI-NEXT: v_readlane_b32 s13, v22, 55
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v22, 56
+; SI-NEXT: v_readlane_b32 s13, v22, 56
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -9155,9 +9112,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v22, 45
-; SI-NEXT: v_readlane_b32 s17, v22, 51
-; SI-NEXT: v_readlane_b32 s15, v22, 57
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -9925,7 +9879,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_readlane_b32 s16, v33, 22
; VI-NEXT: v_mov_b32_e32 v3, s66
-; VI-NEXT: v_readlane_b32 s17, v33, 1
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; VI-NEXT: v_mov_b32_e32 v2, s16
; VI-NEXT: v_perm_b32 v3, s87, v3, v1
@@ -9941,7 +9894,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s65
-; VI-NEXT: v_readlane_b32 s17, v33, 3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
; VI-NEXT: v_mov_b32_e32 v2, s78
; VI-NEXT: v_perm_b32 v3, s70, v3, v1
@@ -9957,19 +9909,18 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s71
-; VI-NEXT: v_readlane_b32 s17, v33, 5
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
; VI-NEXT: v_mov_b32_e32 v2, s69
; VI-NEXT: v_perm_b32 v3, s77, v3, v1
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s16, v33, 6
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s16
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; VI-NEXT: v_mov_b32_e32 v2, s76
-; VI-NEXT: v_perm_b32 v3, s55, v3, v1
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
+; VI-NEXT: v_readlane_b32 s14, v33, 6
+; VI-NEXT: v_mov_b32_e32 v3, s14
+; VI-NEXT: v_perm_b32 v3, s55, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s54
@@ -9978,13 +9929,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s75, v3, v1
; VI-NEXT: v_perm_b32 v2, s15, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s14, v33, 8
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s14
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
; VI-NEXT: v_mov_b32_e32 v2, s72
-; VI-NEXT: v_perm_b32 v3, s73, v3, v1
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
+; VI-NEXT: v_readlane_b32 s12, v33, 8
+; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_perm_b32 v3, s73, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s63
@@ -9993,13 +9944,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s62, v3, v1
; VI-NEXT: v_perm_b32 v2, s13, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s12, v33, 10
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
; VI-NEXT: v_mov_b32_e32 v2, s51
-; VI-NEXT: v_perm_b32 v3, s85, v3, v1
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
+; VI-NEXT: v_readlane_b32 s10, v33, 10
+; VI-NEXT: v_mov_b32_e32 v3, s10
+; VI-NEXT: v_perm_b32 v3, s85, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s50
@@ -10008,13 +9959,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s61, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s10, v33, 12
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s10
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; VI-NEXT: v_mov_b32_e32 v2, s58
-; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
+; VI-NEXT: v_readlane_b32 s8, v33, 12
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s64
@@ -10023,13 +9974,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s83, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s8
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
; VI-NEXT: v_mov_b32_e32 v2, s84
-; VI-NEXT: v_perm_b32 v3, s56, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
+; VI-NEXT: v_readlane_b32 s6, v33, 14
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_perm_b32 v3, s56, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s81
@@ -10038,13 +9989,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s80, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s6, v33, 16
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; VI-NEXT: v_mov_b32_e32 v2, s47
-; VI-NEXT: v_perm_b32 v3, s46, v3, v1
; VI-NEXT: v_perm_b32 v2, s4, v2, v1
+; VI-NEXT: v_readlane_b32 s4, v33, 16
+; VI-NEXT: v_mov_b32_e32 v3, s4
+; VI-NEXT: v_perm_b32 v3, s46, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
@@ -10055,12 +10006,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s17, v33, 7
-; VI-NEXT: v_readlane_b32 s15, v33, 9
-; VI-NEXT: v_readlane_b32 s13, v33, 11
-; VI-NEXT: v_readlane_b32 s11, v33, 13
-; VI-NEXT: v_readlane_b32 s9, v33, 15
-; VI-NEXT: v_readlane_b32 s7, v33, 17
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -10688,7 +10633,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s16, v30, 25
; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
; GFX9-NEXT: v_perm_b32 v12, s25, v15, v1
-; GFX9-NEXT: v_readlane_b32 s17, v30, 1
; GFX9-NEXT: v_mov_b32_e32 v15, s16
; GFX9-NEXT: v_readlane_b32 s16, v30, 23
; GFX9-NEXT: v_mov_b32_e32 v20, s16
@@ -10777,7 +10721,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v20
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s69
-; GFX9-NEXT: v_readlane_b32 s17, v30, 3
; GFX9-NEXT: v_mov_b32_e32 v28, s88
; GFX9-NEXT: v_perm_b32 v3, s85, v3, v1
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
@@ -10793,7 +10736,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s77
-; GFX9-NEXT: v_readlane_b32 s17, v30, 5
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; GFX9-NEXT: v_mov_b32_e32 v2, s68
; GFX9-NEXT: v_perm_b32 v3, s78, v3, v1
@@ -10809,7 +10751,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s74
-; GFX9-NEXT: v_readlane_b32 s17, v30, 7
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
; GFX9-NEXT: v_mov_b32_e32 v2, s75
; GFX9-NEXT: v_perm_b32 v3, s80, v3, v1
@@ -10825,19 +10766,18 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s63
-; GFX9-NEXT: v_readlane_b32 s17, v30, 9
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
; GFX9-NEXT: v_mov_b32_e32 v2, s66
; GFX9-NEXT: v_perm_b32 v3, s72, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s41, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 10
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; GFX9-NEXT: v_mov_b32_e32 v2, s55
-; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s14, v2, v1
+; GFX9-NEXT: v_readlane_b32 s14, v30, 10
+; GFX9-NEXT: v_mov_b32_e32 v3, s14
+; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s60
@@ -10846,13 +10786,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s65, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s15, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 12
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
; GFX9-NEXT: v_mov_b32_e32 v2, s67
-; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s12, v2, v1
+; GFX9-NEXT: v_readlane_b32 s12, v30, 12
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
+; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s51
@@ -10861,13 +10801,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s52, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s13, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 14
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
; GFX9-NEXT: v_mov_b32_e32 v2, s50
-; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s10, v2, v1
+; GFX9-NEXT: v_readlane_b32 s10, v30, 14
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
+; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s57
@@ -10876,13 +10816,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s56, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s11, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 16
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s10
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; GFX9-NEXT: v_mov_b32_e32 v2, s38
-; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s8, v2, v1
+; GFX9-NEXT: v_readlane_b32 s8, v30, 16
+; GFX9-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s98
@@ -10905,13 +10845,13 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s84, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s7, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 18
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; GFX9-NEXT: v_mov_b32_e32 v2, s83
-; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v1
+; GFX9-NEXT: v_readlane_b32 s4, v30, 18
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
+; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
@@ -10922,11 +10862,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
; GFX9-NEXT: v_readlane_b32 s30, v29, 34
-; GFX9-NEXT: v_readlane_b32 s17, v30, 11
-; GFX9-NEXT: v_readlane_b32 s15, v30, 13
-; GFX9-NEXT: v_readlane_b32 s13, v30, 15
-; GFX9-NEXT: v_readlane_b32 s11, v30, 17
-; GFX9-NEXT: v_readlane_b32 s7, v30, 19
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s31, v29, 35
; GFX9-NEXT: v_readlane_b32 s99, v29, 33
@@ -11571,8 +11506,8 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v5, v7, v5
; GFX11-NEXT: v_perm_b32 v6, s102, s101, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
-; GFX11-NEXT: v_readlane_b32 s1, v26, 1
; GFX11-NEXT: v_perm_b32 v14, s7, s36, v1
+; GFX11-NEXT: v_readlane_b32 s102, v24, 4
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v20
@@ -11594,7 +11529,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v7, s23, s100, v1
; GFX11-NEXT: v_perm_b32 v8, s97, s96, v1
; GFX11-NEXT: v_readlane_b32 s0, v26, 2
-; GFX11-NEXT: v_readlane_b32 s1, v26, 3
; GFX11-NEXT: v_or_b32_e32 v4, v9, v10
; GFX11-NEXT: v_perm_b32 v9, s24, s98, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
@@ -11602,7 +11536,6 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v12, s86, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v7, v6
; GFX11-NEXT: v_readlane_b32 s0, v26, 4
-; GFX11-NEXT: v_readlane_b32 s1, v26, 5
; GFX11-NEXT: v_or_b32_e32 v6, v9, v10
; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
; GFX11-NEXT: v_perm_b32 v8, s26, s85, v1
@@ -11620,31 +11553,27 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v10, s29, s67, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-NEXT: v_perm_b32 v12, s66, s0, v1
-; GFX11-NEXT: v_readlane_b32 s1, v26, 7
; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_perm_b32 v4, s40, s65, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
; GFX11-NEXT: v_perm_b32 v10, s55, s54, v1
; GFX11-NEXT: v_readlane_b32 s0, v26, 8
-; GFX11-NEXT: v_readlane_b32 s1, v26, 9
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:48
+; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
; GFX11-NEXT: v_perm_b32 v5, s41, s64, v1
; GFX11-NEXT: v_perm_b32 v11, s62, s0, v1
-; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX11-NEXT: v_readlane_b32 s0, v26, 10
-; GFX11-NEXT: v_readlane_b32 s1, v26, 11
; GFX11-NEXT: v_perm_b32 v8, s14, s63, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX11-NEXT: v_perm_b32 v10, s15, s52, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
; GFX11-NEXT: v_perm_b32 v12, s61, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
; GFX11-NEXT: v_readlane_b32 s0, v26, 12
-; GFX11-NEXT: v_readlane_b32 s1, v26, 13
; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_perm_b32 v8, s12, s58, v1
@@ -11669,33 +11598,33 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
; GFX11-NEXT: v_perm_b32 v10, s39, s43, v1
; GFX11-NEXT: v_perm_b32 v11, s42, s72, v1
-; GFX11-NEXT: v_readlane_b32 s1, v26, 15
; GFX11-NEXT: v_perm_b32 v12, s6, s38, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 16
; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
; GFX11-NEXT: v_perm_b32 v5, s9, s45, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-NEXT: v_readlane_b32 s0, v26, 16
-; GFX11-NEXT: v_readlane_b32 s1, v26, 17
-; GFX11-NEXT: v_readlane_b32 s102, v24, 4
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
-; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s101, v24, 3
; GFX11-NEXT: v_readlane_b32 s100, v24, 2
; GFX11-NEXT: v_readlane_b32 s99, v24, 1
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
+; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s98, v24, 0
+; GFX11-NEXT: v_readlane_b32 s97, v23, 31
+; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_readlane_b32 s87, v23, 29
; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
; GFX11-NEXT: v_perm_b32 v12, s31, s0, v1
-; GFX11-NEXT: v_readlane_b32 s97, v23, 31
-; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_readlane_b32 s86, v23, 28
+; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
; GFX11-NEXT: v_perm_b32 v13, s4, s34, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT: v_readlane_b32 s87, v23, 29
-; GFX11-NEXT: v_readlane_b32 s86, v23, 28
-; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_readlane_b32 s84, v23, 26
+; GFX11-NEXT: v_readlane_b32 s83, v23, 25
+; GFX11-NEXT: v_readlane_b32 s82, v23, 24
+; GFX11-NEXT: v_readlane_b32 s81, v23, 23
; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
; GFX11-NEXT: v_perm_b32 v13, s104, s103, v1
; GFX11-NEXT: v_perm_b32 v1, s5, s30, v1
@@ -11704,17 +11633,14 @@ define inreg <128 x i8> @bitcast_v32i32_to_v128i8_scalar(<32 x i32> inreg %a, i3
; GFX11-NEXT: v_readlane_b32 s104, v24, 6
; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX11-NEXT: v_readlane_b32 s103, v24, 5
-; GFX11-NEXT: v_readlane_b32 s83, v23, 25
-; GFX11-NEXT: v_readlane_b32 s82, v23, 24
-; GFX11-NEXT: v_readlane_b32 s81, v23, 23
+; GFX11-NEXT: v_readlane_b32 s80, v23, 22
+; GFX11-NEXT: v_readlane_b32 s71, v23, 21
+; GFX11-NEXT: v_readlane_b32 s69, v23, 19
; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
; GFX11-NEXT: s_clause 0x2
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
-; GFX11-NEXT: v_readlane_b32 s80, v23, 22
-; GFX11-NEXT: v_readlane_b32 s71, v23, 21
-; GFX11-NEXT: v_readlane_b32 s69, v23, 19
; GFX11-NEXT: v_readlane_b32 s67, v23, 17
; GFX11-NEXT: v_readlane_b32 s66, v23, 16
; GFX11-NEXT: v_readlane_b32 s65, v23, 15
@@ -26493,6 +26419,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -26501,10 +26430,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v37, v36
; SI-NEXT: v_mov_b32_e32 v36, v47
; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v40, v41
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v1, v43
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
@@ -26551,10 +26479,13 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
; SI-NEXT: v_mov_b32_e32 v32, v59
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
@@ -26578,11 +26509,11 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(3)
@@ -26606,10 +26537,8 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
@@ -26649,6 +26578,9 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
; SI-NEXT: v_mov_b32_e32 v39, v62
+; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -26656,38 +26588,27 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v21, v22
; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v47, v36
; SI-NEXT: v_mov_b32_e32 v36, v37
; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_mov_b32_e32 v45, v44
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
@@ -26742,54 +26663,54 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: .LBB19_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_scc1 .LBB19_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -26797,176 +26718,152 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -26974,26 +26871,26 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
@@ -27002,26 +26899,14 @@ define inreg <32 x i32> @bitcast_v64bf16_to_v32i32_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
@@ -40938,60 +40823,55 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: .LBB36_4: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
+; VI-NEXT: v_perm_b32 v13, v13, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v7, v7, v38, s4
+; VI-NEXT: v_perm_b32 v11, v11, v48, s4
; VI-NEXT: v_perm_b32 v10, v10, v37, s4
; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v13, v13, v35, s4
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v8, v8, v33, s4
; VI-NEXT: v_perm_b32 v33, v49, v53, s4
-; VI-NEXT: v_perm_b32 v11, v11, v48, s4
-; VI-NEXT: v_perm_b32 v3, v3, v34, s4
-; VI-NEXT: v_perm_b32 v34, v47, v39, s4
-; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v55, v59, v41, s4
; VI-NEXT: v_perm_b32 v1, v1, v52, s4
; VI-NEXT: v_perm_b32 v2, v2, v58, s4
; VI-NEXT: v_perm_b32 v51, v51, v40, s4
+; VI-NEXT: v_perm_b32 v3, v3, v34, s4
; VI-NEXT: v_perm_b32 v4, v4, v57, s4
+; VI-NEXT: v_perm_b32 v34, v47, v39, s4
; VI-NEXT: v_perm_b32 v5, v5, v56, s4
; VI-NEXT: v_perm_b32 v6, v6, v46, s4
+; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v9, v9, v45, s4
; VI-NEXT: v_perm_b32 v12, v12, v63, s4
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v36, v36, v37, s4
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v14, v14, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v35, v38, v35, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v15, v15, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v14, v14, v35, s4
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v35, v35, v48, s4
+; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v38, v38, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v36, v36, v37, s4
+; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v17, v17, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v18, v18, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v48, v48, v49, s4
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v55
; VI-NEXT: v_or_b32_e32 v1, v1, v49
@@ -41093,12 +40973,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v21, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41113,12 +40993,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41133,12 +41013,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41153,12 +41033,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41173,12 +41053,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v29, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41193,12 +41073,12 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v31, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -41856,61 +41736,55 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v11, v11, v36, s4
; GFX9-NEXT: v_perm_b32 v12, v12, v35, s4
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v34, v38, v39, s4
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v33, s4
; GFX9-NEXT: v_perm_b32 v33, v50, v54, s4
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v55, v58, v41, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v59, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
; GFX9-NEXT: v_perm_b32 v2, v2, v52, s4
-; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
+; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v52, v57, v40, s4
; GFX9-NEXT: v_perm_b32 v5, v5, v56, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v47, s4
; GFX9-NEXT: v_perm_b32 v8, v8, v46, s4
+; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v45, s4
; GFX9-NEXT: v_perm_b32 v10, v10, v44, s4
; GFX9-NEXT: v_perm_b32 v13, v13, v60, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v35, v36, v35, s4
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v39, v36, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v15, v15, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
+; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v39, v39, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v17, v17, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v50, v50, v53, s4
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v50, v50, v51, s4
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GFX9-NEXT: v_perm_b32 v1, v49, v43, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
@@ -42000,10 +41874,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v21, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -42021,10 +41894,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -42042,10 +41914,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -42063,10 +41934,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
@@ -42084,10 +41954,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
@@ -42105,10 +41974,9 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -43600,6 +43468,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s5, 1
; SI-NEXT: s_cbranch_scc1 .LBB37_5
; SI-NEXT: ; %bb.4: ; %cmp.true
+; SI-NEXT: v_add_f32_e64 v1, s60, 1.0
+; SI-NEXT: v_add_f32_e64 v24, s24, 1.0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_readfirstlane_b32 s24, v1
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_add_f32_e64 v1, s58, 1.0
; SI-NEXT: v_add_f32_e64 v22, s26, 1.0
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
@@ -43625,49 +43498,48 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s42, v2
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v1
-; SI-NEXT: v_add_f32_e64 v15, s73, 1.0
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v13
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v15
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v13
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
-; SI-NEXT: v_add_f32_e64 v17, s75, 1.0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; SI-NEXT: v_add_f32_e64 v15, s73, 1.0
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v15
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v13
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v15
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
-; SI-NEXT: v_add_f32_e64 v19, s29, 1.0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; SI-NEXT: v_add_f32_e64 v17, s75, 1.0
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v17
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v15
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v19
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v17
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v19
-; SI-NEXT: v_add_f32_e64 v21, s27, 1.0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
+; SI-NEXT: v_add_f32_e64 v19, s29, 1.0
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v19
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v17
+; SI-NEXT: v_add_f32_e64 v7, s57, 1.0
+; SI-NEXT: v_add_f32_e64 v5, s47, 1.0
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v21
-; SI-NEXT: v_readfirstlane_b32 s43, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v19
+; SI-NEXT: v_readfirstlane_b32 s29, v7
+; SI-NEXT: v_readfirstlane_b32 s41, v5
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v19
; SI-NEXT: v_add_f32_e64 v31, s17, 1.0
-; SI-NEXT: v_add_f32_e64 v35, s16, 1.0
+; SI-NEXT: v_add_f32_e64 v32, s16, 1.0
; SI-NEXT: v_add_f32_e64 v29, s19, 1.0
; SI-NEXT: v_add_f32_e64 v30, s18, 1.0
; SI-NEXT: v_add_f32_e64 v27, s21, 1.0
@@ -43675,23 +43547,22 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_add_f32_e64 v25, s23, 1.0
; SI-NEXT: v_add_f32_e64 v26, s22, 1.0
; SI-NEXT: v_add_f32_e64 v23, s25, 1.0
-; SI-NEXT: v_add_f32_e64 v24, s24, 1.0
+; SI-NEXT: v_add_f32_e64 v21, s27, 1.0
; SI-NEXT: v_add_f32_e64 v18, s74, 1.0
; SI-NEXT: v_add_f32_e64 v16, s72, 1.0
; SI-NEXT: v_add_f32_e64 v14, s62, 1.0
; SI-NEXT: v_add_f32_e64 v11, s61, 1.0
-; SI-NEXT: v_add_f32_e64 v12, s60, 1.0
; SI-NEXT: v_add_f32_e64 v9, s59, 1.0
-; SI-NEXT: v_add_f32_e64 v7, s57, 1.0
-; SI-NEXT: v_add_f32_e64 v5, s47, 1.0
; SI-NEXT: v_add_f32_e64 v3, s45, 1.0
-; SI-NEXT: s_lshr_b64 s[58:59], s[42:43], 24
-; SI-NEXT: s_lshr_b64 s[56:57], s[42:43], 16
-; SI-NEXT: s_lshr_b64 s[42:43], s[42:43], 8
+; SI-NEXT: v_readfirstlane_b32 s43, v1
+; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 24
+; SI-NEXT: s_lshr_b64 s[76:77], s[40:41], 16
+; SI-NEXT: s_lshr_b64 s[78:79], s[40:41], 8
+; SI-NEXT: s_lshr_b64 s[40:41], s[28:29], 16
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v21
-; SI-NEXT: v_readfirstlane_b32 s16, v35
+; SI-NEXT: v_lshrrev_b32_e32 v2, 8, v19
+; SI-NEXT: v_readfirstlane_b32 s16, v32
; SI-NEXT: v_readfirstlane_b32 s17, v31
; SI-NEXT: v_readfirstlane_b32 s10, v30
; SI-NEXT: v_readfirstlane_b32 s11, v29
@@ -43711,31 +43582,27 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readfirstlane_b32 s21, v15
; SI-NEXT: v_readfirstlane_b32 s22, v14
; SI-NEXT: v_readfirstlane_b32 s23, v13
-; SI-NEXT: v_readfirstlane_b32 s24, v12
; SI-NEXT: v_readfirstlane_b32 s25, v11
; SI-NEXT: v_readfirstlane_b32 s27, v9
-; SI-NEXT: v_readfirstlane_b32 s29, v7
-; SI-NEXT: v_readfirstlane_b32 s41, v5
; SI-NEXT: v_readfirstlane_b32 s45, v3
-; SI-NEXT: v_writelane_b32 v61, s42, 30
+; SI-NEXT: s_lshr_b64 s[88:89], s[42:43], 24
+; SI-NEXT: v_writelane_b32 v61, s40, 14
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v23
-; SI-NEXT: v_writelane_b32 v61, s43, 31
-; SI-NEXT: s_lshr_b64 s[42:43], s[44:45], 24
-; SI-NEXT: s_lshr_b64 s[60:61], s[44:45], 16
-; SI-NEXT: s_lshr_b64 s[44:45], s[44:45], 8
-; SI-NEXT: s_lshr_b64 s[62:63], s[40:41], 24
-; SI-NEXT: s_lshr_b64 s[72:73], s[40:41], 16
-; SI-NEXT: s_lshr_b64 s[74:75], s[40:41], 8
-; SI-NEXT: s_lshr_b64 s[76:77], s[28:29], 24
-; SI-NEXT: s_lshr_b64 s[78:79], s[28:29], 16
+; SI-NEXT: v_lshrrev_b32_e32 v2, 24, v21
+; SI-NEXT: s_lshr_b64 s[56:57], s[42:43], 16
+; SI-NEXT: s_lshr_b64 s[58:59], s[42:43], 8
+; SI-NEXT: s_lshr_b64 s[60:61], s[44:45], 24
+; SI-NEXT: s_lshr_b64 s[62:63], s[44:45], 16
+; SI-NEXT: s_lshr_b64 s[72:73], s[44:45], 8
+; SI-NEXT: s_lshr_b64 s[46:47], s[28:29], 24
+; SI-NEXT: v_writelane_b32 v61, s41, 15
; SI-NEXT: s_lshr_b64 s[28:29], s[28:29], 8
-; SI-NEXT: s_lshr_b64 s[88:89], s[26:27], 24
+; SI-NEXT: s_lshr_b64 s[42:43], s[26:27], 24
; SI-NEXT: s_lshr_b64 s[90:91], s[26:27], 16
; SI-NEXT: s_lshr_b64 s[26:27], s[26:27], 8
-; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 24
-; SI-NEXT: s_lshr_b64 s[46:47], s[24:25], 16
+; SI-NEXT: s_lshr_b64 s[44:45], s[24:25], 24
+; SI-NEXT: s_lshr_b64 s[92:93], s[24:25], 16
; SI-NEXT: s_lshr_b64 s[24:25], s[24:25], 8
; SI-NEXT: s_lshr_b64 s[94:95], s[22:23], 24
; SI-NEXT: s_lshr_b64 vcc, s[22:23], 16
@@ -43767,183 +43634,231 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_lshr_b64 s[12:13], s[16:17], 24
; SI-NEXT: s_lshr_b64 s[14:15], s[16:17], 16
; SI-NEXT: s_lshr_b64 s[6:7], s[16:17], 8
-; SI-NEXT: s_mov_b32 s16, s58
-; SI-NEXT: v_lshrrev_b32_e32 v40, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v41, 8, v1
-; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v44, 8, v3
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v5
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v5
-; SI-NEXT: v_lshrrev_b32_e32 v47, 8, v5
-; SI-NEXT: v_lshrrev_b32_e32 v56, 24, v7
-; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v7
-; SI-NEXT: v_lshrrev_b32_e32 v58, 8, v7
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v60, 16, v9
-; SI-NEXT: v_lshrrev_b32_e32 v32, 8, v9
+; SI-NEXT: s_mov_b32 s16, s88
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v40, 8, v1
+; SI-NEXT: v_lshrrev_b32_e32 v41, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v43, 8, v3
+; SI-NEXT: v_lshrrev_b32_e32 v44, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v46, 8, v5
+; SI-NEXT: v_lshrrev_b32_e32 v47, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v56, 16, v7
+; SI-NEXT: v_lshrrev_b32_e32 v57, 8, v7
+; SI-NEXT: v_lshrrev_b32_e32 v58, 24, v9
+; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v9
+; SI-NEXT: v_lshrrev_b32_e32 v60, 8, v9
; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v11
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v36, 8, v11
-; SI-NEXT: v_lshrrev_b32_e32 v37, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v13
+; SI-NEXT: v_lshrrev_b32_e32 v35, 8, v11
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v23
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v36, 8, v21
+; SI-NEXT: v_lshrrev_b32_e32 v37, 24, v23
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v23
; SI-NEXT: v_lshrrev_b32_e32 v4, 8, v23
-; SI-NEXT: v_lshrrev_b32_e32 v39, 24, v25
-; SI-NEXT: v_lshrrev_b32_e32 v48, 16, v25
-; SI-NEXT: v_lshrrev_b32_e32 v49, 8, v25
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v27
-; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v27
-; SI-NEXT: v_lshrrev_b32_e32 v51, 8, v27
-; SI-NEXT: v_lshrrev_b32_e32 v8, 24, v29
-; SI-NEXT: v_lshrrev_b32_e32 v52, 16, v29
-; SI-NEXT: v_lshrrev_b32_e32 v10, 8, v29
-; SI-NEXT: v_lshrrev_b32_e32 v53, 24, v31
-; SI-NEXT: v_lshrrev_b32_e32 v54, 16, v31
-; SI-NEXT: v_lshrrev_b32_e32 v55, 8, v31
+; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v25
+; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v25
+; SI-NEXT: v_lshrrev_b32_e32 v48, 8, v25
+; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v27
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v27
+; SI-NEXT: v_lshrrev_b32_e32 v50, 8, v27
+; SI-NEXT: v_lshrrev_b32_e32 v51, 24, v29
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v29
+; SI-NEXT: v_lshrrev_b32_e32 v52, 8, v29
+; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v31
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v31
+; SI-NEXT: v_lshrrev_b32_e32 v54, 8, v31
; SI-NEXT: s_branch .LBB37_6
; SI-NEXT: .LBB37_5:
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_readlane_b32 s5, v61, 36
-; SI-NEXT: v_mov_b32_e32 v55, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 37
; SI-NEXT: v_mov_b32_e32 v54, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 37
+; SI-NEXT: v_mov_b32_e32 v53, s5
; SI-NEXT: v_readlane_b32 s5, v61, 38
-; SI-NEXT: v_mov_b32_e32 v10, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 39
; SI-NEXT: v_mov_b32_e32 v52, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 39
+; SI-NEXT: v_mov_b32_e32 v10, s5
; SI-NEXT: v_readlane_b32 s5, v61, 40
-; SI-NEXT: v_mov_b32_e32 v8, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 41
; SI-NEXT: v_mov_b32_e32 v51, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 42
+; SI-NEXT: v_readlane_b32 s5, v61, 41
; SI-NEXT: v_mov_b32_e32 v50, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 42
+; SI-NEXT: v_mov_b32_e32 v8, s5
; SI-NEXT: v_readlane_b32 s5, v61, 43
-; SI-NEXT: v_mov_b32_e32 v6, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 44
; SI-NEXT: v_mov_b32_e32 v49, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 45
+; SI-NEXT: v_readlane_b32 s5, v61, 44
; SI-NEXT: v_mov_b32_e32 v48, s5
-; SI-NEXT: v_readlane_b32 s5, v61, 46
+; SI-NEXT: v_readlane_b32 s5, v61, 45
; SI-NEXT: v_mov_b32_e32 v39, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 46
+; SI-NEXT: v_mov_b32_e32 v6, s5
; SI-NEXT: v_readlane_b32 s5, v61, 47
; SI-NEXT: v_mov_b32_e32 v4, s5
; SI-NEXT: v_readlane_b32 s5, v61, 48
-; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: v_mov_b32_e32 v38, s5
; SI-NEXT: v_readlane_b32 s5, v61, 49
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v37, s5
; SI-NEXT: v_readlane_b32 s5, v61, 50
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v36, s5
; SI-NEXT: v_readlane_b32 s5, v61, 51
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v2, s5
; SI-NEXT: v_readlane_b32 s5, v61, 52
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 53
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 54
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 55
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 56
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 57
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 58
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 59
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 60
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 61
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 62
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v61, 63
-; SI-NEXT: v_mov_b32_e32 v38, s5
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v62, 0
-; SI-NEXT: v_mov_b32_e32 v37, s5
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v14, s5
; SI-NEXT: v_readlane_b32 s5, v62, 1
-; SI-NEXT: v_mov_b32_e32 v36, s5
+; SI-NEXT: v_mov_b32_e32 v35, s5
; SI-NEXT: v_readlane_b32 s5, v62, 2
; SI-NEXT: v_mov_b32_e32 v34, s5
; SI-NEXT: v_readlane_b32 s5, v62, 3
; SI-NEXT: v_mov_b32_e32 v33, s5
; SI-NEXT: v_readlane_b32 s5, v62, 4
-; SI-NEXT: v_mov_b32_e32 v32, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 5
; SI-NEXT: v_mov_b32_e32 v60, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 6
+; SI-NEXT: v_readlane_b32 s5, v62, 5
; SI-NEXT: v_mov_b32_e32 v59, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 7
+; SI-NEXT: v_readlane_b32 s5, v62, 6
; SI-NEXT: v_mov_b32_e32 v58, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 8
+; SI-NEXT: v_readlane_b32 s5, v62, 7
; SI-NEXT: v_mov_b32_e32 v57, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 9
+; SI-NEXT: v_readlane_b32 s5, v62, 8
; SI-NEXT: v_mov_b32_e32 v56, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 10
+; SI-NEXT: v_readlane_b32 s5, v62, 9
; SI-NEXT: v_mov_b32_e32 v47, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 11
+; SI-NEXT: v_readlane_b32 s5, v62, 10
; SI-NEXT: v_mov_b32_e32 v46, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 12
+; SI-NEXT: v_readlane_b32 s5, v62, 11
; SI-NEXT: v_mov_b32_e32 v45, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 13
-; SI-NEXT: v_mov_b32_e32 v1, s58
+; SI-NEXT: v_readlane_b32 s5, v62, 12
; SI-NEXT: v_mov_b32_e32 v44, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 13
+; SI-NEXT: v_mov_b32_e32 v43, s5
; SI-NEXT: v_readlane_b32 s5, v62, 14
+; SI-NEXT: v_mov_b32_e32 v42, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 15
+; SI-NEXT: v_mov_b32_e32 v41, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 16
+; SI-NEXT: v_mov_b32_e32 v40, s5
+; SI-NEXT: v_readlane_b32 s5, v62, 17
+; SI-NEXT: v_mov_b32_e32 v1, s60
+; SI-NEXT: v_mov_b32_e32 v55, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 34
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v1, s58
+; SI-NEXT: s_mov_b32 s16, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 32
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s56
-; SI-NEXT: v_mov_b32_e32 v43, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 15
+; SI-NEXT: s_mov_b32 s56, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 30
+; SI-NEXT: s_mov_b32 s58, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 28
+; SI-NEXT: s_mov_b32 s60, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 26
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v14, s62
+; SI-NEXT: s_mov_b32 s62, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 24
+; SI-NEXT: v_mov_b32_e32 v16, s72
+; SI-NEXT: s_mov_b32 s72, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 22
+; SI-NEXT: v_mov_b32_e32 v18, s74
+; SI-NEXT: s_mov_b32 s74, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 20
+; SI-NEXT: s_mov_b32 s76, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 18
+; SI-NEXT: v_mov_b32_e32 v12, s78
+; SI-NEXT: s_mov_b32 s78, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 16
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s46
-; SI-NEXT: v_mov_b32_e32 v42, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 16
+; SI-NEXT: s_mov_b32 s46, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 14
+; SI-NEXT: v_mov_b32_e32 v30, s18
+; SI-NEXT: s_mov_b32 s18, s5
+; SI-NEXT: v_mov_b32_e32 v29, s19
+; SI-NEXT: v_writelane_b32 v61, s18, 14
+; SI-NEXT: v_writelane_b32 v61, s19, 15
+; SI-NEXT: v_readlane_b32 s5, v61, 12
+; SI-NEXT: v_mov_b32_e32 v20, s28
+; SI-NEXT: s_mov_b32 s28, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 10
+; SI-NEXT: v_mov_b32_e32 v22, s26
+; SI-NEXT: s_mov_b32 s26, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 8
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s44
-; SI-NEXT: v_mov_b32_e32 v41, s5
-; SI-NEXT: v_readlane_b32 s5, v62, 17
-; SI-NEXT: v_mov_b32_e32 v5, s47
+; SI-NEXT: s_mov_b32 s44, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 6
+; SI-NEXT: v_mov_b32_e32 v24, s24
+; SI-NEXT: s_mov_b32 s24, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 4
+; SI-NEXT: v_mov_b32_e32 v26, s22
+; SI-NEXT: s_mov_b32 s22, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 2
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s42
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s5
-; SI-NEXT: v_readlane_b32 s46, v61, 34
-; SI-NEXT: v_mov_b32_e32 v29, s19
+; SI-NEXT: v_mov_b32_e32 v28, s20
+; SI-NEXT: s_mov_b32 s20, s5
+; SI-NEXT: v_readlane_b32 s5, v61, 0
+; SI-NEXT: v_mov_b32_e32 v31, s17
; SI-NEXT: v_mov_b32_e32 v27, s21
; SI-NEXT: v_mov_b32_e32 v25, s23
; SI-NEXT: v_mov_b32_e32 v23, s25
@@ -43953,170 +43868,150 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_mov_b32_e32 v15, s73
; SI-NEXT: v_mov_b32_e32 v13, s63
; SI-NEXT: v_mov_b32_e32 v11, s61
+; SI-NEXT: v_mov_b32_e32 v9, s59
; SI-NEXT: v_mov_b32_e32 v7, s57
+; SI-NEXT: v_mov_b32_e32 v5, s47
; SI-NEXT: v_mov_b32_e32 v3, s45
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s43
-; SI-NEXT: v_mov_b32_e32 v53, s78
-; SI-NEXT: v_mov_b32_e32 v40, s79
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v12, s60
-; SI-NEXT: v_mov_b32_e32 v14, s62
-; SI-NEXT: v_mov_b32_e32 v16, s72
-; SI-NEXT: v_mov_b32_e32 v18, s74
-; SI-NEXT: v_mov_b32_e32 v20, s28
-; SI-NEXT: v_mov_b32_e32 v22, s26
-; SI-NEXT: v_mov_b32_e32 v24, s24
-; SI-NEXT: v_mov_b32_e32 v26, s22
-; SI-NEXT: v_mov_b32_e32 v28, s20
-; SI-NEXT: v_mov_b32_e32 v30, s18
-; SI-NEXT: s_mov_b32 s16, s46
-; SI-NEXT: v_readlane_b32 s56, v61, 32
-; SI-NEXT: v_readlane_b32 s42, v61, 28
-; SI-NEXT: v_readlane_b32 s60, v61, 26
-; SI-NEXT: v_readlane_b32 s44, v61, 24
-; SI-NEXT: v_readlane_b32 s62, v61, 22
-; SI-NEXT: v_readlane_b32 s72, v61, 20
-; SI-NEXT: v_readlane_b32 s74, v61, 18
-; SI-NEXT: v_readlane_b32 s76, v61, 16
-; SI-NEXT: v_readlane_b32 s78, v61, 14
-; SI-NEXT: v_readlane_b32 s28, v61, 12
-; SI-NEXT: v_readlane_b32 s26, v61, 10
-; SI-NEXT: s_mov_b32 s46, s92
-; SI-NEXT: v_readlane_b32 s92, v61, 8
-; SI-NEXT: v_readlane_b32 s24, v61, 6
-; SI-NEXT: v_readlane_b32 s22, v61, 4
-; SI-NEXT: v_readlane_b32 s20, v61, 2
-; SI-NEXT: v_readlane_b32 s18, v61, 0
-; SI-NEXT: v_mov_b32_e32 v31, s17
-; SI-NEXT: v_mov_b32_e32 v9, s59
-; SI-NEXT: v_readlane_b32 s47, v61, 35
-; SI-NEXT: v_readlane_b32 s57, v61, 33
-; SI-NEXT: v_readlane_b32 s43, v61, 29
-; SI-NEXT: v_readlane_b32 s61, v61, 27
-; SI-NEXT: v_readlane_b32 s45, v61, 25
-; SI-NEXT: v_readlane_b32 s63, v61, 23
-; SI-NEXT: v_readlane_b32 s73, v61, 21
-; SI-NEXT: v_readlane_b32 s75, v61, 19
-; SI-NEXT: v_readlane_b32 s77, v61, 17
-; SI-NEXT: v_readlane_b32 s79, v61, 15
-; SI-NEXT: v_readlane_b32 s29, v61, 13
-; SI-NEXT: v_readlane_b32 s27, v61, 11
-; SI-NEXT: v_readlane_b32 s93, v61, 9
-; SI-NEXT: v_readlane_b32 s25, v61, 7
-; SI-NEXT: v_readlane_b32 s23, v61, 5
-; SI-NEXT: v_readlane_b32 s21, v61, 3
-; SI-NEXT: v_readlane_b32 s19, v61, 1
+; SI-NEXT: v_mov_b32_e32 v55, s79
+; SI-NEXT: s_mov_b32 s42, s88
+; SI-NEXT: s_mov_b32 s18, s5
; SI-NEXT: .LBB37_6: ; %end
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v35, 0xff, v35
+; SI-NEXT: v_and_b32_e32 v32, 0xff, v32
; SI-NEXT: s_lshl_b32 s5, s6, 8
-; SI-NEXT: v_or_b32_e32 v35, s5, v35
+; SI-NEXT: v_or_b32_e32 v32, s5, v32
; SI-NEXT: s_and_b32 s5, s14, 0xff
-; SI-NEXT: v_and_b32_e32 v29, 0xff, v29
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v10
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s7, s12, 24
-; SI-NEXT: v_or_b32_e32 v10, v29, v10
-; SI-NEXT: v_and_b32_e32 v29, 0xff, v52
-; SI-NEXT: v_and_b32_e32 v35, 0xffff, v35
+; SI-NEXT: v_and_b32_e32 v32, 0xffff, v32
; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v8
-; SI-NEXT: v_or_b32_e32 v35, s5, v35
-; SI-NEXT: v_and_b32_e32 v30, 0xff, v30
+; SI-NEXT: v_or_b32_e32 v32, s5, v32
+; SI-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen
+; SI-NEXT: v_and_b32_e32 v31, 0xff, v31
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 8, v54
+; SI-NEXT: v_or_b32_e32 v31, v31, v32
+; SI-NEXT: v_and_b32_e32 v32, 0xff, v53
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v12
+; SI-NEXT: v_or_b32_e32 v12, v12, v32
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; SI-NEXT: v_or_b32_e32 v12, v31, v12
+; SI-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v30
; SI-NEXT: s_lshl_b32 s5, s10, 8
-; SI-NEXT: v_or_b32_e32 v8, v8, v29
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; SI-NEXT: v_or_b32_e32 v30, s5, v30
+; SI-NEXT: v_or_b32_e32 v12, s5, v12
; SI-NEXT: s_and_b32 s5, s8, 0xff
-; SI-NEXT: v_or_b32_e32 v8, v10, v8
; SI-NEXT: s_lshl_b32 s5, s5, 16
; SI-NEXT: s_lshl_b32 s6, s40, 24
-; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: s_or_b32 s5, s6, s5
+; SI-NEXT: v_or_b32_e32 v12, s5, v12
+; SI-NEXT: buffer_store_dword v12, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v12, 0xff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v29, 8, v52
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v10
+; SI-NEXT: v_or_b32_e32 v12, v12, v29
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: v_lshlrev_b32_e32 v29, 24, v51
+; SI-NEXT: v_or_b32_e32 v10, v29, v10
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_or_b32_e32 v10, v12, v10
+; SI-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v28
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v28
; SI-NEXT: s_lshl_b32 s4, s4, 8
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_or_b32_e32 v8, s4, v8
+; SI-NEXT: v_or_b32_e32 v10, s4, v10
; SI-NEXT: s_and_b32 s4, s98, 0xff
-; SI-NEXT: v_or_b32_e32 v30, s5, v30
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s96, 24
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_or_b32_e32 v10, s4, v10
+; SI-NEXT: buffer_store_dword v10, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v12, 8, v50
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v8
+; SI-NEXT: v_or_b32_e32 v10, v10, v12
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v49
+; SI-NEXT: v_or_b32_e32 v8, v12, v8
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_or_b32_e32 v8, v10, v8
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v26
+; SI-NEXT: s_lshl_b32 s4, s86, 8
+; SI-NEXT: v_or_b32_e32 v8, s4, v8
+; SI-NEXT: s_and_b32 s4, s84, 0xff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s5, s82, 24
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v8, s4, v8
-; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: buffer_store_dword v8, v0, s[0:3], 0 offen offset:24
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v27
-; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v51
+; SI-NEXT: v_and_b32_e32 v8, 0xff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v10, 8, v48
; SI-NEXT: v_or_b32_e32 v8, v8, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xff, v50
+; SI-NEXT: v_and_b32_e32 v10, 0xff, v39
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_or_b32_e32 v6, v6, v10
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:20
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v26
-; SI-NEXT: s_lshl_b32 s4, s86, 8
-; SI-NEXT: v_or_b32_e32 v6, s4, v6
-; SI-NEXT: s_and_b32 s4, s84, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: s_lshl_b32 s5, s82, 24
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_or_b32_e32 v6, s4, v6
-; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:24
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xff, v25
-; SI-NEXT: v_lshlrev_b32_e32 v8, 8, v49
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: v_and_b32_e32 v8, 0xff, v48
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; SI-NEXT: v_lshlrev_b32_e32 v10, 24, v39
-; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: buffer_store_dword v35, v0, s[0:3], 0 offen
-; SI-NEXT: v_and_b32_e32 v31, 0xff, v31
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v35, 8, v55
; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:28
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v6, 0xff, v24
; SI-NEXT: s_lshl_b32 s4, s80, 8
-; SI-NEXT: v_or_b32_e32 v31, v31, v35
-; SI-NEXT: v_and_b32_e32 v35, 0xff, v54
; SI-NEXT: v_or_b32_e32 v6, s4, v6
; SI-NEXT: s_and_b32 s4, s70, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v35, 16, v35
-; SI-NEXT: v_lshlrev_b32_e32 v53, 24, v53
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: s_lshl_b32 s5, s68, 24
-; SI-NEXT: v_or_b32_e32 v35, v53, v35
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_or_b32_e32 v31, v31, v35
; SI-NEXT: v_or_b32_e32 v6, s4, v6
-; SI-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:4
-; SI-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:8
; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v6, 0xff, v23
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v38
+; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-NEXT: v_lshlrev_b32_e32 v8, 24, v37
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: v_or_b32_e32 v6, v8, v6
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:36
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v22
+; SI-NEXT: s_lshl_b32 s4, s66, 8
+; SI-NEXT: v_or_b32_e32 v4, s4, v4
+; SI-NEXT: s_and_b32 s4, s64, 0xff
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_lshl_b32 s5, s54, 24
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_or_b32_e32 v4, s4, v4
+; SI-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v6, 8, v36
+; SI-NEXT: v_or_b32_e32 v4, v4, v6
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; SI-NEXT: s_lshl_b32 s4, s66, 8
-; SI-NEXT: s_lshl_b32 s5, s54, 24
+; SI-NEXT: s_lshl_b32 s4, s52, 8
+; SI-NEXT: s_lshl_b32 s5, s48, 24
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_readlane_b32 s99, v63, 33
; SI-NEXT: v_readlane_b32 s98, v63, 32
@@ -44137,11 +44032,14 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s67, v63, 17
; SI-NEXT: v_readlane_b32 s66, v63, 16
; SI-NEXT: v_readlane_b32 s65, v63, 15
+; SI-NEXT: v_readlane_b32 s64, v63, 14
; SI-NEXT: v_readlane_b32 s55, v63, 13
; SI-NEXT: v_readlane_b32 s54, v63, 12
; SI-NEXT: v_readlane_b32 s53, v63, 11
+; SI-NEXT: v_readlane_b32 s52, v63, 10
; SI-NEXT: v_readlane_b32 s51, v63, 9
; SI-NEXT: v_readlane_b32 s49, v63, 7
+; SI-NEXT: v_readlane_b32 s48, v63, 6
; SI-NEXT: v_readlane_b32 s39, v63, 5
; SI-NEXT: v_readlane_b32 s37, v63, 3
; SI-NEXT: v_readlane_b32 s35, v63, 1
@@ -44149,37 +44047,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_or_b32_e32 v2, v6, v2
; SI-NEXT: v_or_b32_e32 v2, v4, v2
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v22
-; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s64, 0xff
-; SI-NEXT: s_lshl_b32 s4, s4, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v21
-; SI-NEXT: s_lshl_b32 s4, s52, 8
-; SI-NEXT: s_lshl_b32 s5, s48, 24
-; SI-NEXT: v_readlane_b32 s64, v63, 14
-; SI-NEXT: v_readlane_b32 s52, v63, 10
-; SI-NEXT: v_readlane_b32 s48, v63, 6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v20
@@ -44190,18 +44057,18 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v2, s4, v2
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v19
; SI-NEXT: s_lshl_b32 s4, s18, 8
; SI-NEXT: s_lshl_b32 s5, s36, 24
; SI-NEXT: v_readlane_b32 s50, v63, 8
; SI-NEXT: v_readlane_b32 s36, v63, 2
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
@@ -44220,7 +44087,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v2, s4, v2
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v17
; SI-NEXT: s_lshl_b32 s4, s20, 8
@@ -44228,11 +44095,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s30, v63, 34
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s38, v63, 4
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
@@ -44251,17 +44118,17 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v2, s4, v2
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v15
; SI-NEXT: s_lshl_b32 s4, s22, 8
; SI-NEXT: s_lshl_b32 s5, s94, 24
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
@@ -44280,25 +44147,33 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v2, s4, v2
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v37
; SI-NEXT: s_lshl_b32 s4, s24, 8
-; SI-NEXT: s_lshl_b32 s5, s92, 24
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_lshl_b32 s5, s44, 24
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v38
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v12
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v35
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v33
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s46, 0xff
+; SI-NEXT: s_and_b32 s4, s92, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44306,11 +44181,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:80
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v11
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v36
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v4, 0xff, v34
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v33
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
@@ -44318,9 +44191,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s26, 8
-; SI-NEXT: s_lshl_b32 s5, s88, 24
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v32
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v59
+; SI-NEXT: s_lshl_b32 s5, s42, 24
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v60
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v58
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
@@ -44333,7 +44206,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v9
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v60
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v59
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
@@ -44342,13 +44215,14 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_lshl_b32 s4, s28, 8
-; SI-NEXT: s_lshl_b32 s5, s76, 24
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v58
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v56
+; SI-NEXT: s_lshl_b32 s5, s46, 24
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v57
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v47
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s78, 0xff
+; SI-NEXT: v_readlane_b32 s4, v61, 14
+; SI-NEXT: s_and_b32 s4, s4, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44357,7 +44231,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v7
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v57
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v56
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
@@ -44365,13 +44239,13 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s4, s74, 8
-; SI-NEXT: s_lshl_b32 s5, s62, 24
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v47
+; SI-NEXT: s_lshl_b32 s4, s78, 8
+; SI-NEXT: s_lshl_b32 s5, s74, 24
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v46
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s72, 0xff
+; SI-NEXT: s_and_b32 s4, s76, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44380,22 +44254,22 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v5
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v46
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v45
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v45
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v44
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v5, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s4, s44, 8
-; SI-NEXT: s_lshl_b32 s5, s42, 24
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v42
+; SI-NEXT: s_lshl_b32 s4, s72, 8
+; SI-NEXT: s_lshl_b32 s5, s60, 24
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v41
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
; SI-NEXT: v_or_b32_e32 v2, s4, v2
-; SI-NEXT: s_and_b32 s4, s60, 0xff
+; SI-NEXT: s_and_b32 s4, s62, 0xff
; SI-NEXT: s_lshl_b32 s4, s4, 16
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_or_b32 s4, s5, s4
@@ -44403,9 +44277,9 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:112
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v44
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v43
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v43
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v42
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v3, v4, v3
@@ -44413,9 +44287,7 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: v_readlane_b32 s4, v61, 30
-; SI-NEXT: s_lshl_b32 s4, s4, 8
-; SI-NEXT: v_readlane_b32 s5, v61, 31
+; SI-NEXT: s_lshl_b32 s4, s58, 8
; SI-NEXT: s_lshl_b32 s5, s16, 24
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
@@ -44426,11 +44298,11 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_or_b32_e32 v2, s4, v2
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v41
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v40
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v40
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v55
; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -45326,20 +45198,18 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: .LBB37_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v17, v36, v17, s4
-; VI-NEXT: v_perm_b32 v32, v32, v49, s4
; VI-NEXT: v_perm_b32 v36, v15, v54, s4
-; VI-NEXT: v_perm_b32 v30, v30, v48, s4
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v16, v16, v39, s4
+; VI-NEXT: v_perm_b32 v30, v30, v48, s4
+; VI-NEXT: v_perm_b32 v32, v32, v49, s4
; VI-NEXT: v_perm_b32 v33, v33, v50, s4
-; VI-NEXT: v_perm_b32 v18, v35, v18, s4
-; VI-NEXT: v_perm_b32 v35, v51, v40, s4
; VI-NEXT: v_perm_b32 v55, v55, v42, s4
; VI-NEXT: v_perm_b32 v20, v37, v20, s4
; VI-NEXT: v_perm_b32 v19, v38, v19, s4
; VI-NEXT: v_perm_b32 v37, v52, v41, s4
+; VI-NEXT: v_perm_b32 v18, v35, v18, s4
+; VI-NEXT: v_perm_b32 v35, v51, v40, s4
; VI-NEXT: v_perm_b32 v34, v34, v61, s4
; VI-NEXT: v_perm_b32 v31, v31, v60, s4
; VI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -45383,30 +45253,28 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: v_readlane_b32 s36, v63, 2
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v39, v47, v48, s4
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v39, v47, v15, s4
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v48, v44, v15, s4
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v24, v24, v15, s4
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_perm_b32 v48, v44, v48, s4
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v49, v15, v49, s4
+; VI-NEXT: v_perm_b32 v49, v49, v15, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v21, v21, v15, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v22, v22, v15, s4
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v15, v15, v50, s4
+; VI-NEXT: v_perm_b32 v15, v50, v15, s4
; VI-NEXT: v_lshlrev_b32_e32 v50, 16, v55
; VI-NEXT: v_or_b32_e32 v20, v20, v50
; VI-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen
@@ -45515,7 +45383,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v11, v11, v13, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v13, v14, v13, s4
@@ -45536,7 +45403,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v9, v9, v11, s4
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v11, v12, v11, s4
@@ -45557,7 +45423,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v7, v7, v9, s4
; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v9, v10, v9, s4
@@ -45578,7 +45443,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v5, v5, v7, s4
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v7, v8, v7, s4
@@ -45599,7 +45463,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v3, v3, v5, s4
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v5, v6, v5, s4
@@ -45620,7 +45483,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v3, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v3, v4, v3, s4
@@ -46525,21 +46387,17 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: .LBB37_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v32, v32, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v19, v49, v19, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v30, v30, v38, s4
; GFX9-NEXT: v_perm_b32 v16, v34, v16, s4
; GFX9-NEXT: v_perm_b32 v34, v50, v39, s4
-; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v30, v30, v38, s4
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v18, v35, v18, s4
-; GFX9-NEXT: v_perm_b32 v35, v51, v40, s4
+; GFX9-NEXT: v_perm_b32 v19, v49, v19, s4
; GFX9-NEXT: v_perm_b32 v55, v55, v42, s4
; GFX9-NEXT: v_perm_b32 v20, v48, v20, s4
; GFX9-NEXT: v_perm_b32 v48, v52, v41, s4
+; GFX9-NEXT: v_perm_b32 v18, v35, v18, s4
; GFX9-NEXT: v_perm_b32 v17, v36, v17, s4
+; GFX9-NEXT: v_perm_b32 v35, v51, v40, s4
; GFX9-NEXT: v_perm_b32 v33, v33, v61, s4
; GFX9-NEXT: v_perm_b32 v31, v31, v60, s4
; GFX9-NEXT: v_perm_b32 v36, v59, v54, s4
@@ -46586,27 +46444,28 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v24, v24, v15, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v39, v44, v49, s4
-; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v38, v47, v38, s4
+; GFX9-NEXT: v_perm_b32 v38, v47, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v39, v44, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v24, v24, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v49, v15, v49, s4
+; GFX9-NEXT: v_perm_b32 v49, v49, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v21, v21, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v22, v22, v15, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v15, v15, v50, s4
+; GFX9-NEXT: v_perm_b32 v15, v50, v15, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v50, 16, v55
; GFX9-NEXT: v_or_b32_e32 v20, v20, v50
; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen
@@ -46726,7 +46585,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v11, v13, s4
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v13, v14, v13, s4
@@ -46747,7 +46605,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v9, v11, s4
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v12, v11, s4
@@ -46768,7 +46625,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v7, v9, s4
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v10, v9, s4
@@ -46789,7 +46645,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v5, v7, s4
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v8, v7, s4
@@ -46810,7 +46665,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v3, v5, s4
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v6, v5, s4
@@ -46831,7 +46685,6 @@ define inreg <128 x i8> @bitcast_v32f32_to_v128i8_scalar(<32 x float> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4
@@ -62702,6 +62555,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -62710,10 +62566,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v37, v36
; SI-NEXT: v_mov_b32_e32 v36, v47
; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v40, v41
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v1, v43
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
@@ -62760,10 +62615,13 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
; SI-NEXT: v_mov_b32_e32 v32, v59
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
@@ -62787,11 +62645,11 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(3)
@@ -62815,10 +62673,8 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
@@ -62858,6 +62714,9 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
; SI-NEXT: v_mov_b32_e32 v39, v62
+; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -62865,38 +62724,27 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v21, v22
; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v47, v36
; SI-NEXT: v_mov_b32_e32 v36, v37
; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_mov_b32_e32 v45, v44
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
@@ -62951,54 +62799,54 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: .LBB43_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_scc1 .LBB43_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -63006,176 +62854,152 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -63183,26 +63007,26 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
@@ -63211,26 +63035,14 @@ define inreg <32 x float> @bitcast_v64bf16_to_v32f32_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
@@ -67076,7 +66888,6 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
; SI-NEXT: v_or_b32_e32 v29, v29, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
; SI-NEXT: v_readlane_b32 s30, v63, 20
@@ -67101,7 +66912,7 @@ define inreg <64 x half> @bitcast_v32f32_to_v64f16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v30, v30, v32
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v55
@@ -70439,7 +70250,6 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
; SI-NEXT: v_or_b32_e32 v29, v29, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_and_b32_e32 v31, 0xffff, v31
; SI-NEXT: v_readlane_b32 s30, v63, 20
@@ -70464,7 +70274,7 @@ define inreg <64 x i16> @bitcast_v32f32_to_v64i16_scalar(<32 x float> inreg %a,
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v30, v30, v32
; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v55
@@ -75776,60 +75586,55 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: .LBB56_4: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
+; VI-NEXT: v_perm_b32 v13, v13, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v7, v7, v38, s4
+; VI-NEXT: v_perm_b32 v11, v11, v48, s4
; VI-NEXT: v_perm_b32 v10, v10, v37, s4
; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v13, v13, v35, s4
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v8, v8, v33, s4
; VI-NEXT: v_perm_b32 v33, v49, v53, s4
-; VI-NEXT: v_perm_b32 v11, v11, v48, s4
-; VI-NEXT: v_perm_b32 v3, v3, v34, s4
-; VI-NEXT: v_perm_b32 v34, v47, v39, s4
-; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v55, v59, v41, s4
; VI-NEXT: v_perm_b32 v1, v1, v52, s4
; VI-NEXT: v_perm_b32 v2, v2, v58, s4
; VI-NEXT: v_perm_b32 v51, v51, v40, s4
+; VI-NEXT: v_perm_b32 v3, v3, v34, s4
; VI-NEXT: v_perm_b32 v4, v4, v57, s4
+; VI-NEXT: v_perm_b32 v34, v47, v39, s4
; VI-NEXT: v_perm_b32 v5, v5, v56, s4
; VI-NEXT: v_perm_b32 v6, v6, v46, s4
+; VI-NEXT: v_perm_b32 v39, v50, v54, s4
; VI-NEXT: v_perm_b32 v9, v9, v45, s4
; VI-NEXT: v_perm_b32 v12, v12, v63, s4
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v36, v36, v37, s4
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v14, v14, v35, s4
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v35, v38, v35, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v15, v15, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v14, v14, v35, s4
-; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v35, v35, v48, s4
+; VI-NEXT: v_perm_b32 v16, v16, v38, s4
; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v38, v38, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v36, v36, v37, s4
+; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v17, v17, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v37, v60, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v18, v18, v48, s4
; VI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v48, v48, v49, s4
; VI-NEXT: v_lshlrev_b32_e32 v49, 16, v55
; VI-NEXT: v_or_b32_e32 v1, v1, v49
@@ -75931,12 +75736,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v21, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -75951,12 +75756,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -75971,12 +75776,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -75991,12 +75796,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -76011,12 +75816,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v29, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -76031,12 +75836,12 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v31, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -76694,61 +76499,55 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v11, v11, v36, s4
; GFX9-NEXT: v_perm_b32 v12, v12, v35, s4
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v34, v38, v39, s4
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v6, v6, v33, s4
; GFX9-NEXT: v_perm_b32 v33, v50, v54, s4
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v55, v58, v41, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v59, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v1
; GFX9-NEXT: v_perm_b32 v2, v2, v52, s4
-; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
+; GFX9-NEXT: v_perm_b32 v4, v4, v51, s4
; GFX9-NEXT: v_perm_b32 v52, v57, v40, s4
; GFX9-NEXT: v_perm_b32 v5, v5, v56, s4
; GFX9-NEXT: v_perm_b32 v7, v7, v47, s4
; GFX9-NEXT: v_perm_b32 v8, v8, v46, s4
+; GFX9-NEXT: v_perm_b32 v37, v37, v53, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v45, s4
; GFX9-NEXT: v_perm_b32 v10, v10, v44, s4
; GFX9-NEXT: v_perm_b32 v13, v13, v60, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v35, v36, v35, s4
; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v39, v36, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v15, v15, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v14, v14, v36, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v36, v36, v50, s4
+; GFX9-NEXT: v_perm_b32 v16, v16, v39, s4
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v39, v39, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v17, v17, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v63, v38, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v50, v50, v53, s4
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v50, v50, v51, s4
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GFX9-NEXT: v_perm_b32 v1, v49, v43, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
@@ -76838,10 +76637,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v21, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -76859,10 +76657,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -76880,10 +76677,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -76901,10 +76697,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
@@ -76922,10 +76717,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
@@ -76943,10 +76737,9 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -78795,15 +78588,14 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 0
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s22, 0xff
; SI-NEXT: s_lshl_b32 s17, s74, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 1
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 0
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 2
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78820,23 +78612,20 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 7
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 6
-; SI-NEXT: v_readlane_b32 s19, v22, 3
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 4
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 5
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 6
+; SI-NEXT: v_readlane_b32 s17, v22, 4
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s24, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 7
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 6
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 8
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78855,19 +78644,17 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readlane_b32 s18, v21, 3
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: v_readlane_b32 s19, v22, 9
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 10
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s26, 0xff
; SI-NEXT: s_lshl_b32 s17, s46, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 11
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 10
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 12
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78881,23 +78668,20 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s73, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s72, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 13
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s63, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 14
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 15
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 16
+; SI-NEXT: v_readlane_b32 s17, v22, 14
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s28, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 17
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 16
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 18
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78911,23 +78695,20 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s62, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s87, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 19
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s56, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 20
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 21
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 22
+; SI-NEXT: v_readlane_b32 s17, v22, 20
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 23
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 22
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 24
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78941,23 +78722,20 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s85, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s86, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 25
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s55, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 26
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 27
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 28
+; SI-NEXT: v_readlane_b32 s17, v22, 26
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 29
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 28
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 30
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -78971,23 +78749,20 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_lshl_b32 s17, s83, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s84, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 31
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s54, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 32
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 33
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 34
+; SI-NEXT: v_readlane_b32 s17, v22, 32
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 35
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 34
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 36
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -79011,16 +78786,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_readlane_b32 s16, v22, 38
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v22, 39
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 37
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v22, 40
-; SI-NEXT: v_readlane_b32 s17, v22, 41
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 42
+; SI-NEXT: v_readlane_b32 s17, v22, 42
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_or_b32 s14, s14, s16
@@ -79041,15 +78813,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_readlane_b32 s14, v22, 44
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 45
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v22, 46
-; SI-NEXT: v_readlane_b32 s15, v22, 47
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 48
+; SI-NEXT: v_readlane_b32 s15, v22, 48
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -79071,7 +78841,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readlane_b32 s12, v22, 50
; SI-NEXT: s_and_b32 s10, s10, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 8
-; SI-NEXT: v_readlane_b32 s13, v22, 51
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: s_and_b32 s12, s60, 0xff
; SI-NEXT: s_lshl_b32 s12, s12, 16
@@ -79096,15 +78865,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_readlane_b32 s10, v22, 52
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v22, 53
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
; SI-NEXT: v_readlane_b32 s10, v22, 54
-; SI-NEXT: v_readlane_b32 s11, v22, 55
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v22, 56
+; SI-NEXT: v_readlane_b32 s11, v22, 56
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_or_b32 s8, s8, s10
@@ -79126,7 +78893,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_readlane_b32 s8, v22, 58
; SI-NEXT: s_and_b32 s6, s6, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s9, v22, 59
; SI-NEXT: s_or_b32 s6, s6, s8
; SI-NEXT: s_and_b32 s8, s98, 0xff
; SI-NEXT: s_lshl_b32 s8, s8, 16
@@ -79151,15 +78917,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: v_readlane_b32 s6, v22, 60
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s7, v22, 61
; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: v_readlane_b32 s6, v21, 0
-; SI-NEXT: v_readlane_b32 s7, v21, 1
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v22, 62
+; SI-NEXT: v_readlane_b32 s7, v22, 62
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -79179,10 +78943,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v22, 43
-; SI-NEXT: v_readlane_b32 s17, v22, 49
-; SI-NEXT: v_readlane_b32 s13, v22, 57
-; SI-NEXT: v_readlane_b32 s9, v22, 63
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -79950,7 +79710,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_readlane_b32 s16, v33, 22
; VI-NEXT: v_mov_b32_e32 v3, s66
-; VI-NEXT: v_readlane_b32 s17, v33, 1
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; VI-NEXT: v_mov_b32_e32 v2, s16
; VI-NEXT: v_perm_b32 v3, s87, v3, v1
@@ -79966,7 +79725,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s65
-; VI-NEXT: v_readlane_b32 s17, v33, 3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
; VI-NEXT: v_mov_b32_e32 v2, s78
; VI-NEXT: v_perm_b32 v3, s70, v3, v1
@@ -79982,19 +79740,18 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s71
-; VI-NEXT: v_readlane_b32 s17, v33, 5
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
; VI-NEXT: v_mov_b32_e32 v2, s69
; VI-NEXT: v_perm_b32 v3, s77, v3, v1
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s16, v33, 6
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s16
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; VI-NEXT: v_mov_b32_e32 v2, s76
-; VI-NEXT: v_perm_b32 v3, s55, v3, v1
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
+; VI-NEXT: v_readlane_b32 s14, v33, 6
+; VI-NEXT: v_mov_b32_e32 v3, s14
+; VI-NEXT: v_perm_b32 v3, s55, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s54
@@ -80003,13 +79760,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s75, v3, v1
; VI-NEXT: v_perm_b32 v2, s15, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s14, v33, 8
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s14
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
; VI-NEXT: v_mov_b32_e32 v2, s72
-; VI-NEXT: v_perm_b32 v3, s73, v3, v1
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
+; VI-NEXT: v_readlane_b32 s12, v33, 8
+; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_perm_b32 v3, s73, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s63
@@ -80018,13 +79775,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s62, v3, v1
; VI-NEXT: v_perm_b32 v2, s13, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s12, v33, 10
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
; VI-NEXT: v_mov_b32_e32 v2, s51
-; VI-NEXT: v_perm_b32 v3, s85, v3, v1
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
+; VI-NEXT: v_readlane_b32 s10, v33, 10
+; VI-NEXT: v_mov_b32_e32 v3, s10
+; VI-NEXT: v_perm_b32 v3, s85, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s50
@@ -80033,13 +79790,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s61, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s10, v33, 12
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s10
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; VI-NEXT: v_mov_b32_e32 v2, s58
-; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
+; VI-NEXT: v_readlane_b32 s8, v33, 12
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: v_perm_b32 v3, s59, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s64
@@ -80048,13 +79805,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s83, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s8, v33, 14
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s8
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
; VI-NEXT: v_mov_b32_e32 v2, s84
-; VI-NEXT: v_perm_b32 v3, s56, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
+; VI-NEXT: v_readlane_b32 s6, v33, 14
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_perm_b32 v3, s56, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s81
@@ -80063,13 +79820,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s80, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s6, v33, 16
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; VI-NEXT: v_mov_b32_e32 v2, s47
-; VI-NEXT: v_perm_b32 v3, s46, v3, v1
; VI-NEXT: v_perm_b32 v2, s4, v2, v1
+; VI-NEXT: v_readlane_b32 s4, v33, 16
+; VI-NEXT: v_mov_b32_e32 v3, s4
+; VI-NEXT: v_perm_b32 v3, s46, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
@@ -80080,12 +79837,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s17, v33, 7
-; VI-NEXT: v_readlane_b32 s15, v33, 9
-; VI-NEXT: v_readlane_b32 s13, v33, 11
-; VI-NEXT: v_readlane_b32 s11, v33, 13
-; VI-NEXT: v_readlane_b32 s9, v33, 15
-; VI-NEXT: v_readlane_b32 s7, v33, 17
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -80713,7 +80464,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s16, v30, 25
; GFX9-NEXT: v_perm_b32 v13, s24, v12, v1
; GFX9-NEXT: v_perm_b32 v12, s25, v15, v1
-; GFX9-NEXT: v_readlane_b32 s17, v30, 1
; GFX9-NEXT: v_mov_b32_e32 v15, s16
; GFX9-NEXT: v_readlane_b32 s16, v30, 23
; GFX9-NEXT: v_mov_b32_e32 v20, s16
@@ -80802,7 +80552,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v20
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s69
-; GFX9-NEXT: v_readlane_b32 s17, v30, 3
; GFX9-NEXT: v_mov_b32_e32 v28, s88
; GFX9-NEXT: v_perm_b32 v3, s85, v3, v1
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
@@ -80818,7 +80567,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s77
-; GFX9-NEXT: v_readlane_b32 s17, v30, 5
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; GFX9-NEXT: v_mov_b32_e32 v2, s68
; GFX9-NEXT: v_perm_b32 v3, s78, v3, v1
@@ -80834,7 +80582,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s74
-; GFX9-NEXT: v_readlane_b32 s17, v30, 7
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
; GFX9-NEXT: v_mov_b32_e32 v2, s75
; GFX9-NEXT: v_perm_b32 v3, s80, v3, v1
@@ -80850,19 +80597,18 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s63
-; GFX9-NEXT: v_readlane_b32 s17, v30, 9
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
; GFX9-NEXT: v_mov_b32_e32 v2, s66
; GFX9-NEXT: v_perm_b32 v3, s72, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s41, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s16, v30, 10
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s16
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; GFX9-NEXT: v_mov_b32_e32 v2, s55
-; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s14, v2, v1
+; GFX9-NEXT: v_readlane_b32 s14, v30, 10
+; GFX9-NEXT: v_mov_b32_e32 v3, s14
+; GFX9-NEXT: v_perm_b32 v3, s62, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s60
@@ -80871,13 +80617,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s65, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s15, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s14, v30, 12
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s14
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
; GFX9-NEXT: v_mov_b32_e32 v2, s67
-; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s12, v2, v1
+; GFX9-NEXT: v_readlane_b32 s12, v30, 12
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
+; GFX9-NEXT: v_perm_b32 v3, s58, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s51
@@ -80886,13 +80632,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s52, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s13, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s12, v30, 14
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
; GFX9-NEXT: v_mov_b32_e32 v2, s50
-; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s10, v2, v1
+; GFX9-NEXT: v_readlane_b32 s10, v30, 14
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
+; GFX9-NEXT: v_perm_b32 v3, s48, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s57
@@ -80901,13 +80647,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s56, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s11, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s10, v30, 16
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s10
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; GFX9-NEXT: v_mov_b32_e32 v2, s38
-; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s8, v2, v1
+; GFX9-NEXT: v_readlane_b32 s8, v30, 16
+; GFX9-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-NEXT: v_perm_b32 v3, s39, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: v_mov_b32_e32 v3, s98
@@ -80930,13 +80676,13 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_perm_b32 v3, s84, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s7, v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_readlane_b32 s6, v30, 18
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; GFX9-NEXT: v_mov_b32_e32 v2, s83
-; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_perm_b32 v2, s4, v2, v1
+; GFX9-NEXT: v_readlane_b32 s4, v30, 18
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
+; GFX9-NEXT: v_perm_b32 v3, s81, v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v2, v2, v3
; GFX9-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
@@ -80947,11 +80693,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v1, v2, v1
; GFX9-NEXT: v_readlane_b32 s30, v29, 34
-; GFX9-NEXT: v_readlane_b32 s17, v30, 11
-; GFX9-NEXT: v_readlane_b32 s15, v30, 13
-; GFX9-NEXT: v_readlane_b32 s13, v30, 15
-; GFX9-NEXT: v_readlane_b32 s11, v30, 17
-; GFX9-NEXT: v_readlane_b32 s7, v30, 19
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; GFX9-NEXT: v_readlane_b32 s31, v29, 35
; GFX9-NEXT: v_readlane_b32 s99, v29, 33
@@ -81598,8 +81339,8 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_or_b32_e32 v5, v7, v5
; GFX11-NEXT: v_perm_b32 v6, s102, s101, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v14
-; GFX11-NEXT: v_readlane_b32 s1, v26, 1
; GFX11-NEXT: v_perm_b32 v14, s7, s36, v1
+; GFX11-NEXT: v_readlane_b32 s102, v24, 4
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v8
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v20
@@ -81621,7 +81362,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v7, s23, s100, v1
; GFX11-NEXT: v_perm_b32 v8, s97, s96, v1
; GFX11-NEXT: v_readlane_b32 s0, v26, 2
-; GFX11-NEXT: v_readlane_b32 s1, v26, 3
; GFX11-NEXT: v_or_b32_e32 v4, v9, v10
; GFX11-NEXT: v_perm_b32 v9, s24, s98, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
@@ -81629,7 +81369,6 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v12, s86, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v7, v6
; GFX11-NEXT: v_readlane_b32 s0, v26, 4
-; GFX11-NEXT: v_readlane_b32 s1, v26, 5
; GFX11-NEXT: v_or_b32_e32 v6, v9, v10
; GFX11-NEXT: v_or_b32_e32 v7, v11, v8
; GFX11-NEXT: v_perm_b32 v8, s26, s85, v1
@@ -81647,31 +81386,27 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_perm_b32 v10, s29, s67, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-NEXT: v_perm_b32 v12, s66, s0, v1
-; GFX11-NEXT: v_readlane_b32 s1, v26, 7
; GFX11-NEXT: v_or_b32_e32 v9, v9, v3
; GFX11-NEXT: v_or_b32_e32 v2, v4, v5
-; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_perm_b32 v4, s40, s65, v1
+; GFX11-NEXT: v_or_b32_e32 v3, v10, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
; GFX11-NEXT: v_perm_b32 v10, s55, s54, v1
; GFX11-NEXT: v_readlane_b32 s0, v26, 8
-; GFX11-NEXT: v_readlane_b32 s1, v26, 9
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:48
+; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
; GFX11-NEXT: v_perm_b32 v5, s41, s64, v1
; GFX11-NEXT: v_perm_b32 v11, s62, s0, v1
-; GFX11-NEXT: v_perm_b32 v6, s53, s60, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX11-NEXT: v_readlane_b32 s0, v26, 10
-; GFX11-NEXT: v_readlane_b32 s1, v26, 11
; GFX11-NEXT: v_perm_b32 v8, s14, s63, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX11-NEXT: v_perm_b32 v10, s15, s52, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v11
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v6
; GFX11-NEXT: v_perm_b32 v12, s61, s0, v1
; GFX11-NEXT: v_or_b32_e32 v5, v5, v7
; GFX11-NEXT: v_readlane_b32 s0, v26, 12
-; GFX11-NEXT: v_readlane_b32 s1, v26, 13
; GFX11-NEXT: v_or_b32_e32 v6, v8, v9
; GFX11-NEXT: v_or_b32_e32 v7, v10, v11
; GFX11-NEXT: v_perm_b32 v8, s12, s58, v1
@@ -81696,33 +81431,33 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v12
; GFX11-NEXT: v_perm_b32 v10, s39, s43, v1
; GFX11-NEXT: v_perm_b32 v11, s42, s72, v1
-; GFX11-NEXT: v_readlane_b32 s1, v26, 15
; GFX11-NEXT: v_perm_b32 v12, s6, s38, v1
+; GFX11-NEXT: v_readlane_b32 s0, v26, 16
; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
; GFX11-NEXT: v_perm_b32 v5, s9, s45, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-NEXT: v_readlane_b32 s0, v26, 16
-; GFX11-NEXT: v_readlane_b32 s1, v26, 17
-; GFX11-NEXT: v_readlane_b32 s102, v24, 4
-; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
-; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s101, v24, 3
; GFX11-NEXT: v_readlane_b32 s100, v24, 2
; GFX11-NEXT: v_readlane_b32 s99, v24, 1
+; GFX11-NEXT: v_or_b32_e32 v5, v5, v10
+; GFX11-NEXT: v_perm_b32 v10, s37, s35, v1
; GFX11-NEXT: v_readlane_b32 s98, v24, 0
+; GFX11-NEXT: v_readlane_b32 s97, v23, 31
+; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_readlane_b32 s87, v23, 29
; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v10
; GFX11-NEXT: v_or_b32_e32 v10, v12, v11
; GFX11-NEXT: v_perm_b32 v12, s31, s0, v1
-; GFX11-NEXT: v_readlane_b32 s97, v23, 31
-; GFX11-NEXT: v_readlane_b32 s96, v23, 30
+; GFX11-NEXT: v_readlane_b32 s86, v23, 28
+; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_or_b32_e32 v11, v14, v13
; GFX11-NEXT: v_perm_b32 v13, s4, s34, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-NEXT: v_readlane_b32 s87, v23, 29
-; GFX11-NEXT: v_readlane_b32 s86, v23, 28
-; GFX11-NEXT: v_readlane_b32 s85, v23, 27
; GFX11-NEXT: v_readlane_b32 s84, v23, 26
+; GFX11-NEXT: v_readlane_b32 s83, v23, 25
+; GFX11-NEXT: v_readlane_b32 s82, v23, 24
+; GFX11-NEXT: v_readlane_b32 s81, v23, 23
; GFX11-NEXT: v_or_b32_e32 v12, v13, v12
; GFX11-NEXT: v_perm_b32 v13, s104, s103, v1
; GFX11-NEXT: v_perm_b32 v1, s5, s30, v1
@@ -81731,17 +81466,14 @@ define inreg <128 x i8> @bitcast_v16i64_to_v128i8_scalar(<16 x i64> inreg %a, i3
; GFX11-NEXT: v_readlane_b32 s104, v24, 6
; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; GFX11-NEXT: v_readlane_b32 s103, v24, 5
-; GFX11-NEXT: v_readlane_b32 s83, v23, 25
-; GFX11-NEXT: v_readlane_b32 s82, v23, 24
-; GFX11-NEXT: v_readlane_b32 s81, v23, 23
+; GFX11-NEXT: v_readlane_b32 s80, v23, 22
+; GFX11-NEXT: v_readlane_b32 s71, v23, 21
+; GFX11-NEXT: v_readlane_b32 s69, v23, 19
; GFX11-NEXT: v_or_b32_e32 v13, v1, v13
; GFX11-NEXT: s_clause 0x2
; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:80
; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off offset:96
; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:112
-; GFX11-NEXT: v_readlane_b32 s80, v23, 22
-; GFX11-NEXT: v_readlane_b32 s71, v23, 21
-; GFX11-NEXT: v_readlane_b32 s69, v23, 19
; GFX11-NEXT: v_readlane_b32 s67, v23, 17
; GFX11-NEXT: v_readlane_b32 s66, v23, 16
; GFX11-NEXT: v_readlane_b32 s65, v23, 15
@@ -96501,6 +96233,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -96509,10 +96244,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v37, v36
; SI-NEXT: v_mov_b32_e32 v36, v47
; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v40, v41
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v1, v43
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
@@ -96559,10 +96293,13 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
; SI-NEXT: v_mov_b32_e32 v32, v59
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
@@ -96586,11 +96323,11 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(3)
@@ -96614,10 +96351,8 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
@@ -96657,6 +96392,9 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
; SI-NEXT: v_mov_b32_e32 v39, v62
+; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -96664,38 +96402,27 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_mov_b32_e32 v21, v22
; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v47, v36
; SI-NEXT: v_mov_b32_e32 v36, v37
; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_mov_b32_e32 v45, v44
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
@@ -96750,54 +96477,54 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: .LBB63_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_scc1 .LBB63_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -96805,176 +96532,152 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -96982,26 +96685,26 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
@@ -97010,26 +96713,14 @@ define inreg <16 x i64> @bitcast_v64bf16_to_v16i64_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
@@ -107914,20 +107605,17 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: .LBB72_4: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
-; VI-NEXT: v_perm_b32 v7, v7, v51, s4
-; VI-NEXT: v_perm_b32 v10, v10, v50, s4
-; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v13, v13, v36, s4
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v7, v7, v51, s4
; VI-NEXT: v_perm_b32 v3, v3, v52, s4
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v11, v11, v37, s4
+; VI-NEXT: v_perm_b32 v10, v10, v50, s4
+; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v8, v8, v38, s4
; VI-NEXT: v_perm_b32 v38, v63, v53, s4
; VI-NEXT: v_perm_b32 v55, v57, v41, s4
; VI-NEXT: v_perm_b32 v1, v1, v58, s4
+; VI-NEXT: v_perm_b32 v11, v11, v37, s4
; VI-NEXT: v_perm_b32 v2, v2, v56, s4
; VI-NEXT: v_perm_b32 v35, v35, v40, s4
; VI-NEXT: v_perm_b32 v4, v4, v47, s4
@@ -107937,17 +107625,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v34, v44, v54, s4
; VI-NEXT: v_perm_b32 v9, v9, v43, s4
; VI-NEXT: v_perm_b32 v12, v12, v61, s4
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_perm_b32 v50, v62, v50, s4
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v14, v14, v36, s4
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v37, v59, v51, s4
-; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v14, v14, v36, s4
+; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v36, v36, v51, s4
+; VI-NEXT: v_perm_b32 v36, v51, v36, s4
; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v15, v15, v51, s4
@@ -107955,7 +107638,6 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v16, v16, v51, s4
; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v51, v52, v51, s4
@@ -107963,11 +107645,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v17, v17, v52, s4
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v50, v62, v50, s4
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v18, v18, v52, s4
; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
+; VI-NEXT: v_perm_b32 v37, v59, v37, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v52, v53, v52, s4
; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v55
@@ -108071,13 +107756,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v21, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v21, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -108091,12 +107775,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -108111,12 +107795,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -108131,12 +107815,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -108151,12 +107835,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v29, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -108171,12 +107855,12 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v31, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -108819,17 +108503,16 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: .LBB72_4: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v5, v5, v52, s4
-; GFX9-NEXT: v_perm_b32 v8, v8, v51, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v10, v10, v50, s4
; GFX9-NEXT: v_perm_b32 v34, v34, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v12, v12, v36, s4
+; GFX9-NEXT: v_perm_b32 v10, v10, v50, s4
+; GFX9-NEXT: v_perm_b32 v8, v8, v51, s4
; GFX9-NEXT: v_perm_b32 v35, v35, v41, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v57, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v35, 16, v1
; GFX9-NEXT: v_perm_b32 v2, v2, v53, s4
+; GFX9-NEXT: v_perm_b32 v5, v5, v52, s4
; GFX9-NEXT: v_perm_b32 v53, v47, v40, s4
; GFX9-NEXT: v_perm_b32 v3, v3, v56, s4
; GFX9-NEXT: v_perm_b32 v4, v4, v46, s4
@@ -108838,17 +108521,15 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v7, v7, v44, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v43, s4
; GFX9-NEXT: v_perm_b32 v11, v11, v62, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v33, v33, v51, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v39, v49, v50, s4
-; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v36, v36, v49, s4
+; GFX9-NEXT: v_perm_b32 v33, v33, v39, s4
+; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v39, v49, v39, s4
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v49, v36, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v13, v13, v49, s4
@@ -108856,7 +108537,6 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v14, v14, v49, s4
; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v49, v50, v49, s4
@@ -108867,7 +108547,6 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v16, v16, v50, s4
; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v50, v51, v50, s4
@@ -108878,7 +108557,6 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v51, s4
; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v51, v52, v51, s4
@@ -108965,14 +108643,14 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v21, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v21, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
@@ -108989,10 +108667,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -109010,10 +108687,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -109030,10 +108706,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v58, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
@@ -109051,10 +108726,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
@@ -109072,10 +108746,9 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) #0 {
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -110564,64 +110237,167 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_cmp_lg_u32 s47, 1
; SI-NEXT: s_cbranch_scc1 .LBB73_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: v_add_f64 v[15:16], s[6:7], 1.0
-; SI-NEXT: v_add_f64 v[33:34], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[1:2], s[4:5], 1.0
+; SI-NEXT: v_add_f64 v[3:4], s[6:7], 1.0
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 24
+; SI-NEXT: v_add_f64 v[5:6], s[8:9], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 16
+; SI-NEXT: v_add_f64 v[7:8], s[10:11], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[1:2], 8
+; SI-NEXT: v_add_f64 v[9:10], s[12:13], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 24
+; SI-NEXT: v_add_f64 v[11:12], s[14:15], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 16
+; SI-NEXT: v_add_f64 v[13:14], s[40:41], 1.0
; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[17:18], v[33:34], 24
-; SI-NEXT: v_add_f64 v[44:45], s[12:13], 1.0
-; SI-NEXT: v_add_f64 v[47:48], s[40:41], 1.0
-; SI-NEXT: v_add_f64 v[10:11], s[42:43], 1.0
-; SI-NEXT: v_add_f64 v[36:37], s[24:25], 1.0
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s5, v34
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[33:34], 16
-; SI-NEXT: v_lshr_b64 v[34:35], v[33:34], 8
-; SI-NEXT: v_lshr_b64 v[55:56], v[44:45], 16
-; SI-NEXT: v_readfirstlane_b32 s13, v45
-; SI-NEXT: v_lshr_b64 v[58:59], v[44:45], 24
-; SI-NEXT: v_lshr_b64 v[56:57], v[44:45], 8
-; SI-NEXT: v_lshr_b64 v[45:46], v[47:48], 16
-; SI-NEXT: v_lshr_b64 v[23:24], v[10:11], 24
-; SI-NEXT: v_mov_b32_e32 v35, v36
-; SI-NEXT: v_mov_b32_e32 v36, v37
-; SI-NEXT: v_add_f64 v[1:2], s[22:23], 1.0
-; SI-NEXT: v_add_f64 v[20:21], s[8:9], 1.0
-; SI-NEXT: v_add_f64 v[41:42], s[20:21], 1.0
-; SI-NEXT: v_add_f64 v[28:29], s[10:11], 1.0
-; SI-NEXT: v_add_f64 v[3:4], s[26:27], 1.0
-; SI-NEXT: v_add_f64 v[5:6], s[44:45], 1.0
-; SI-NEXT: v_readfirstlane_b32 s9, v21
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: v_add_f64 v[52:53], s[14:15], 1.0
-; SI-NEXT: v_add_f64 v[49:50], s[18:19], 1.0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[17:18], v[47:48], 8
-; SI-NEXT: v_add_f64 v[13:14], s[28:29], 1.0
-; SI-NEXT: v_add_f64 v[7:8], s[16:17], 1.0
-; SI-NEXT: v_readfirstlane_b32 s15, v53
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: v_readfirstlane_b32 s19, v50
-; SI-NEXT: v_readfirstlane_b32 s21, v42
-; SI-NEXT: v_readfirstlane_b32 s25, v37
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[18:19], v[13:14], 16
-; SI-NEXT: v_readfirstlane_b32 s17, v8
-; SI-NEXT: v_readfirstlane_b32 s23, v2
-; SI-NEXT: v_readfirstlane_b32 s45, v6
-; SI-NEXT: v_readfirstlane_b32 s11, v29
-; SI-NEXT: v_readfirstlane_b32 s41, v48
-; SI-NEXT: v_lshr_b64 v[59:60], v[47:48], 24
-; SI-NEXT: v_readfirstlane_b32 s29, v14
-; SI-NEXT: v_readfirstlane_b32 s43, v11
-; SI-NEXT: v_readfirstlane_b32 s27, v4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[3:4], 8
+; SI-NEXT: v_readfirstlane_b32 s41, v14
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 24
+; SI-NEXT: v_add_f64 v[18:19], s[42:43], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 16
+; SI-NEXT: v_add_f64 v[31:32], s[28:29], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[5:6], 8
+; SI-NEXT: v_add_f64 v[37:38], s[26:27], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 24
+; SI-NEXT: v_add_f64 v[44:45], s[22:23], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 16
+; SI-NEXT: v_readfirstlane_b32 s29, v32
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[7:8], 8
+; SI-NEXT: v_lshr_b64 v[23:24], v[31:32], 24
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 24
+; SI-NEXT: v_lshr_b64 v[29:30], v[31:32], 16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 16
+; SI-NEXT: v_lshr_b64 v[32:33], v[31:32], 8
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[9:10], 8
+; SI-NEXT: v_add_f64 v[25:26], s[44:45], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 24
+; SI-NEXT: v_add_f64 v[53:54], s[24:25], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 16
+; SI-NEXT: v_add_f64 v[57:58], s[20:21], 1.0
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 8
+; SI-NEXT: v_readfirstlane_b32 s27, v38
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[13:14], 24
+; SI-NEXT: v_lshr_b64 v[35:36], v[37:38], 24
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[15:16], v[13:14], 16
+; SI-NEXT: v_lshr_b64 v[47:48], v[37:38], 16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshr_b64 v[14:15], v[13:14], 8
+; SI-NEXT: v_lshr_b64 v[33:34], v[37:38], 8
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 24
+; SI-NEXT: v_lshr_b64 v[38:39], v[44:45], 16
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[39:40], v[44:45], 8
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[18:19], 8
+; SI-NEXT: v_add_f64 v[27:28], s[16:17], 1.0
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_add_f64 v[21:22], s[18:19], 1.0
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[25:26], 24
+; SI-NEXT: v_lshr_b64 v[48:49], v[53:54], 24
+; SI-NEXT: v_lshr_b64 v[40:41], v[57:58], 24
+; SI-NEXT: v_lshr_b64 v[49:50], v[53:54], 16
+; SI-NEXT: v_lshr_b64 v[41:42], v[57:58], 16
+; SI-NEXT: v_readfirstlane_b32 s21, v58
+; SI-NEXT: v_readfirstlane_b32 s25, v54
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[50:51], v[53:54], 8
+; SI-NEXT: v_lshr_b64 v[42:43], v[57:58], 8
+; SI-NEXT: v_lshr_b64 v[54:55], v[21:22], 16
+; SI-NEXT: v_lshr_b64 v[58:59], v[27:28], 24
+; SI-NEXT: v_readfirstlane_b32 s17, v28
+; SI-NEXT: v_readfirstlane_b32 s19, v22
+; SI-NEXT: v_readfirstlane_b32 s23, v45
+; SI-NEXT: v_readfirstlane_b32 s45, v26
+; SI-NEXT: v_readfirstlane_b32 s43, v19
+; SI-NEXT: v_readfirstlane_b32 s15, v12
+; SI-NEXT: v_readfirstlane_b32 s13, v10
+; SI-NEXT: v_readfirstlane_b32 s11, v8
+; SI-NEXT: v_readfirstlane_b32 s9, v6
+; SI-NEXT: v_readfirstlane_b32 s7, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v2
+; SI-NEXT: v_lshr_b64 v[16:17], v[25:26], 16
+; SI-NEXT: v_lshr_b64 v[19:20], v[25:26], 8
+; SI-NEXT: v_lshr_b64 v[51:52], v[44:45], 24
+; SI-NEXT: v_lshr_b64 v[45:46], v[21:22], 24
+; SI-NEXT: v_lshr_b64 v[55:56], v[21:22], 8
+; SI-NEXT: v_lshr_b64 v[59:60], v[27:28], 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[14:15], v[27:28], 8
; SI-NEXT: s_lshr_b32 s10, s5, 24
; SI-NEXT: s_lshr_b32 s12, s5, 16
; SI-NEXT: s_lshr_b32 s14, s5, 8
+; SI-NEXT: s_lshr_b32 s16, s7, 24
+; SI-NEXT: s_lshr_b32 s18, s7, 16
+; SI-NEXT: s_lshr_b32 s20, s7, 8
; SI-NEXT: s_lshr_b32 s22, s9, 24
; SI-NEXT: s_lshr_b32 s24, s9, 16
; SI-NEXT: s_lshr_b32 s26, s9, 8
@@ -110652,132 +110428,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b32 s90, s25, 24
; SI-NEXT: s_lshr_b32 s91, s25, 16
; SI-NEXT: s_lshr_b32 s92, s25, 8
-; SI-NEXT: v_lshr_b64 v[25:26], v[15:16], 8
-; SI-NEXT: v_readfirstlane_b32 s7, v16
-; SI-NEXT: v_mov_b32_e32 v46, v25
-; SI-NEXT: v_lshr_b64 v[24:25], v[35:36], 24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[35:36], 16
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[35:36], 8
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[24:25], v[1:2], 24
-; SI-NEXT: v_lshr_b64 v[38:39], v[15:16], 24
-; SI-NEXT: v_lshr_b64 v[30:31], v[15:16], 16
-; SI-NEXT: v_lshr_b64 v[15:16], v[20:21], 24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[24:25], v[1:2], 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[20:21], 16
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[24:25], v[1:2], 8
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[20:21], 8
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 24
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[28:29], 24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[15:16], v[28:29], 16
-; SI-NEXT: v_lshr_b64 v[21:22], v[3:4], 24
-; SI-NEXT: v_lshr_b64 v[31:32], v[5:6], 24
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_lshr_b64 v[24:25], v[41:42], 8
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[15:16], v[28:29], 8
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[21:22], v[3:4], 16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[31:32], v[5:6], 16
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[39:40], v[52:53], 24
-; SI-NEXT: v_lshr_b64 v[26:27], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[53:54], v[52:53], 8
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(4)
-; SI-NEXT: v_lshr_b64 v[24:25], v[49:50], 24
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[16:17], v[10:11], 16
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[21:22], v[3:4], 8
-; SI-NEXT: v_mov_b32_e32 v54, v38
-; SI-NEXT: v_lshr_b64 v[37:38], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v40, v49
-; SI-NEXT: v_lshr_b64 v[42:43], v[49:50], 8
-; SI-NEXT: v_lshr_b64 v[49:50], v[7:8], 24
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[31:32], v[5:6], 8
-; SI-NEXT: v_mov_b32_e32 v19, v30
-; SI-NEXT: v_mov_b32_e32 v22, v35
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[29:30], v[7:8], 16
-; SI-NEXT: v_mov_b32_e32 v35, v39
-; SI-NEXT: v_mov_b32_e32 v6, v7
-; SI-NEXT: v_mov_b32_e32 v7, v8
-; SI-NEXT: v_mov_b32_e32 v50, v1
-; SI-NEXT: v_mov_b32_e32 v39, v56
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v16
-; SI-NEXT: v_mov_b32_e32 v16, v3
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[31:32], v[13:14], 24
-; SI-NEXT: v_lshr_b64 v[14:15], v[13:14], 8
-; SI-NEXT: v_lshr_b64 v[11:12], v[10:11], 8
-; SI-NEXT: s_lshr_b32 s16, s7, 24
-; SI-NEXT: s_lshr_b32 s18, s7, 16
-; SI-NEXT: s_lshr_b32 s20, s7, 8
; SI-NEXT: s_lshr_b32 s93, s23, 24
; SI-NEXT: s_lshr_b32 s94, s23, 16
; SI-NEXT: s_lshr_b32 s95, s23, 8
@@ -110790,85 +110440,53 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_lshr_b32 s8, s17, 24
; SI-NEXT: s_lshr_b32 s36, s17, 16
; SI-NEXT: s_lshr_b32 s6, s17, 8
-; SI-NEXT: v_mov_b32_e32 v12, v11
-; SI-NEXT: v_mov_b32_e32 v36, v5
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mov_b32_e32 v15, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_mov_b32_e32 v30, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v24
-; SI-NEXT: v_mov_b32_e32 v24, v20
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v20, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v3, v31
-; SI-NEXT: v_lshr_b64 v[31:32], v[6:7], 8
-; SI-NEXT: v_mov_b32_e32 v33, v2
; SI-NEXT: s_branch .LBB73_6
; SI-NEXT: .LBB73_5:
-; SI-NEXT: v_mov_b32_e32 v1, s90
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: v_mov_b32_e32 v2, s68
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s88
-; SI-NEXT: v_mov_b32_e32 v21, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 0
-; SI-NEXT: v_readlane_b32 s47, v62, 1
-; SI-NEXT: v_mov_b32_e32 v53, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 2
-; SI-NEXT: v_readlane_b32 s47, v62, 3
-; SI-NEXT: v_mov_b32_e32 v26, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 4
-; SI-NEXT: v_readlane_b32 s47, v62, 5
-; SI-NEXT: v_mov_b32_e32 v35, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 6
-; SI-NEXT: v_readlane_b32 s47, v62, 7
-; SI-NEXT: v_mov_b32_e32 v39, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 8
-; SI-NEXT: v_readlane_b32 s47, v62, 9
-; SI-NEXT: v_mov_b32_e32 v55, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 10
-; SI-NEXT: v_readlane_b32 s47, v62, 11
-; SI-NEXT: v_mov_b32_e32 v58, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 12
-; SI-NEXT: v_readlane_b32 s47, v62, 13
-; SI-NEXT: v_mov_b32_e32 v3, s82
-; SI-NEXT: v_mov_b32_e32 v31, s34
-; SI-NEXT: v_mov_b32_e32 v29, s30
-; SI-NEXT: v_mov_b32_e32 v49, vcc_lo
-; SI-NEXT: v_mov_b32_e32 v42, s94
-; SI-NEXT: v_mov_b32_e32 v37, s92
-; SI-NEXT: v_mov_b32_e32 v6, s16
-; SI-NEXT: v_mov_b32_e32 v40, s18
-; SI-NEXT: v_mov_b32_e32 v41, s20
-; SI-NEXT: v_mov_b32_e32 v50, s22
-; SI-NEXT: v_mov_b32_e32 v22, s24
-; SI-NEXT: v_mov_b32_e32 v16, s26
-; SI-NEXT: v_mov_b32_e32 v13, s28
-; SI-NEXT: v_mov_b32_e32 v36, s44
-; SI-NEXT: v_mov_b32_e32 v10, s42
-; SI-NEXT: v_mov_b32_e32 v47, s40
-; SI-NEXT: v_mov_b32_e32 v52, s14
-; SI-NEXT: v_mov_b32_e32 v44, s12
-; SI-NEXT: v_mov_b32_e32 v28, s10
-; SI-NEXT: v_mov_b32_e32 v24, s8
-; SI-NEXT: v_mov_b32_e32 v51, s4
-; SI-NEXT: v_mov_b32_e32 v14, s86
-; SI-NEXT: v_mov_b32_e32 v18, s84
-; SI-NEXT: v_mov_b32_e32 v12, s66
-; SI-NEXT: v_mov_b32_e32 v48, s64
-; SI-NEXT: v_mov_b32_e32 v23, s54
-; SI-NEXT: v_mov_b32_e32 v45, s50
-; SI-NEXT: v_mov_b32_e32 v59, s48
-; SI-NEXT: v_mov_b32_e32 v15, s38
+; SI-NEXT: v_mov_b32_e32 v2, s66
+; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 0
+; SI-NEXT: v_mov_b32_e32 v14, s34
+; SI-NEXT: v_mov_b32_e32 v59, s30
+; SI-NEXT: v_mov_b32_e32 v58, vcc_lo
+; SI-NEXT: v_mov_b32_e32 v55, s94
+; SI-NEXT: v_mov_b32_e32 v54, s92
+; SI-NEXT: v_mov_b32_e32 v45, s90
+; SI-NEXT: v_mov_b32_e32 v42, s88
+; SI-NEXT: v_mov_b32_e32 v41, s78
+; SI-NEXT: v_mov_b32_e32 v40, s76
+; SI-NEXT: v_mov_b32_e32 v39, s74
+; SI-NEXT: v_mov_b32_e32 v38, s72
+; SI-NEXT: v_mov_b32_e32 v51, s62
+; SI-NEXT: v_mov_b32_e32 v50, s60
+; SI-NEXT: v_mov_b32_e32 v49, s58
+; SI-NEXT: v_mov_b32_e32 v48, s56
+; SI-NEXT: v_mov_b32_e32 v33, s46
+; SI-NEXT: v_mov_b32_e32 v27, s16
+; SI-NEXT: v_mov_b32_e32 v21, s18
+; SI-NEXT: v_mov_b32_e32 v57, s20
+; SI-NEXT: v_mov_b32_e32 v44, s22
+; SI-NEXT: v_mov_b32_e32 v53, s24
+; SI-NEXT: v_mov_b32_e32 v37, s26
+; SI-NEXT: v_mov_b32_e32 v31, s28
+; SI-NEXT: v_mov_b32_e32 v25, s44
+; SI-NEXT: v_mov_b32_e32 v18, s42
+; SI-NEXT: v_mov_b32_e32 v13, s40
+; SI-NEXT: v_mov_b32_e32 v11, s14
+; SI-NEXT: v_mov_b32_e32 v9, s12
+; SI-NEXT: v_mov_b32_e32 v7, s10
+; SI-NEXT: v_mov_b32_e32 v5, s8
+; SI-NEXT: v_mov_b32_e32 v47, s98
+; SI-NEXT: v_mov_b32_e32 v35, s96
+; SI-NEXT: v_mov_b32_e32 v32, s86
+; SI-NEXT: v_mov_b32_e32 v29, s84
+; SI-NEXT: v_mov_b32_e32 v23, s82
+; SI-NEXT: v_mov_b32_e32 v19, s80
+; SI-NEXT: v_mov_b32_e32 v16, s70
; SI-NEXT: v_readlane_b32 s10, v62, 34
; SI-NEXT: v_readlane_b32 s12, v62, 35
; SI-NEXT: v_readlane_b32 s14, v62, 36
@@ -110882,13 +110500,23 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s40, v62, 44
; SI-NEXT: v_readlane_b32 s42, v62, 45
; SI-NEXT: v_readlane_b32 s44, v62, 46
+; SI-NEXT: v_readlane_b32 s46, v62, 47
+; SI-NEXT: v_readlane_b32 s47, v62, 48
+; SI-NEXT: v_readlane_b32 s56, v62, 49
; SI-NEXT: v_readlane_b32 s57, v62, 50
+; SI-NEXT: v_readlane_b32 s58, v62, 51
; SI-NEXT: v_readlane_b32 s59, v62, 52
+; SI-NEXT: v_readlane_b32 s60, v62, 53
; SI-NEXT: v_readlane_b32 s61, v62, 54
+; SI-NEXT: v_readlane_b32 s62, v62, 55
; SI-NEXT: v_readlane_b32 s63, v62, 56
+; SI-NEXT: v_readlane_b32 s72, v62, 57
; SI-NEXT: v_readlane_b32 s73, v62, 58
+; SI-NEXT: v_readlane_b32 s74, v62, 59
; SI-NEXT: v_readlane_b32 s75, v62, 60
+; SI-NEXT: v_readlane_b32 s76, v62, 61
; SI-NEXT: v_readlane_b32 s77, v62, 62
+; SI-NEXT: v_readlane_b32 s78, v62, 63
; SI-NEXT: v_readlane_b32 s79, v61, 0
; SI-NEXT: v_readlane_b32 s88, v61, 1
; SI-NEXT: v_readlane_b32 s89, v61, 2
@@ -110896,11 +110524,10 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s91, v61, 4
; SI-NEXT: v_readlane_b32 s92, v61, 5
; SI-NEXT: v_readlane_b32 s93, v61, 6
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s78
-; SI-NEXT: v_readlane_b32 s78, v62, 63
+; SI-NEXT: v_mov_b32_e32 v2, s64
; SI-NEXT: v_readlane_b32 s94, v61, 7
; SI-NEXT: v_readlane_b32 s95, v61, 8
; SI-NEXT: v_readlane_b32 vcc_lo, v61, 9
@@ -110911,117 +110538,124 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s35, v61, 14
; SI-NEXT: v_readlane_b32 s8, v61, 15
; SI-NEXT: v_readlane_b32 s36, v61, 16
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_readlane_b32 s6, v61, 17
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s76
-; SI-NEXT: v_readlane_b32 s76, v62, 61
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s54
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s74
-; SI-NEXT: v_readlane_b32 s74, v62, 59
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s52
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s72
-; SI-NEXT: v_readlane_b32 s72, v62, 57
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s50
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s62
-; SI-NEXT: v_readlane_b32 s62, v62, 55
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s48
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s60
-; SI-NEXT: v_readlane_b32 s60, v62, 53
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 2
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s58
-; SI-NEXT: v_readlane_b32 s58, v62, 51
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 4
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s56
-; SI-NEXT: v_readlane_b32 s56, v62, 49
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 6
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s6
-; SI-NEXT: v_readlane_b32 s6, v61, 17
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 8
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s98
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 10
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s96
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 12
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s80
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 14
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s70
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s68
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 18
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 14
-; SI-NEXT: v_readlane_b32 s47, v62, 15
-; SI-NEXT: v_mov_b32_e32 v56, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 16
-; SI-NEXT: v_readlane_b32 s47, v62, 17
-; SI-NEXT: v_mov_b32_e32 v30, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 18
-; SI-NEXT: v_readlane_b32 s47, v62, 19
-; SI-NEXT: v_mov_b32_e32 v8, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 20
-; SI-NEXT: v_readlane_b32 s47, v62, 21
-; SI-NEXT: v_mov_b32_e32 v20, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 22
-; SI-NEXT: v_readlane_b32 s47, v62, 23
-; SI-NEXT: v_mov_b32_e32 v33, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 24
-; SI-NEXT: v_readlane_b32 s47, v62, 25
-; SI-NEXT: v_mov_b32_e32 v46, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 26
-; SI-NEXT: v_readlane_b32 s47, v62, 27
-; SI-NEXT: v_mov_b32_e32 v19, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 28
-; SI-NEXT: v_readlane_b32 s47, v62, 29
-; SI-NEXT: v_mov_b32_e32 v54, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 30
-; SI-NEXT: v_readlane_b32 s47, v62, 31
-; SI-NEXT: v_mov_b32_e32 v34, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 32
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s52
-; SI-NEXT: v_readlane_b32 s47, v62, 33
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 20
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 22
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 24
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 26
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 28
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 30
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_readlane_b32 s4, v62, 32
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s38
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v38, s46
-; SI-NEXT: v_readlane_b32 s46, v62, 47
-; SI-NEXT: v_readlane_b32 s47, v62, 48
; SI-NEXT: .LBB73_6: ; %end
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v6
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v31
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v27
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v14
; SI-NEXT: s_and_b32 s4, s17, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v29
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v59
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s36, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v49
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v58
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s8, 24
@@ -111035,26 +110669,164 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v40
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v42
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v21
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v55
; SI-NEXT: s_and_b32 s4, s19, 0xff
; SI-NEXT: s_lshl_b32 s6, s35, 8
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v37
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v54
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s34, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v45
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, s31, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:12
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v57
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v42
+; SI-NEXT: s_and_b32 s4, s21, 0xff
; SI-NEXT: s_lshl_b32 s6, s30, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v41
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v40
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s8, vcc_lo, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v44
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v39
+; SI-NEXT: s_and_b32 s4, s23, 0xff
+; SI-NEXT: s_lshl_b32 s6, s95, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v38
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s94, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v51
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s93, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:28
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v53
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v50
+; SI-NEXT: s_and_b32 s4, s25, 0xff
+; SI-NEXT: s_lshl_b32 s6, s92, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v49
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s91, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v48
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s90, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v37
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v33
+; SI-NEXT: s_and_b32 s4, s27, 0xff
+; SI-NEXT: s_lshl_b32 s6, s89, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v47
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s88, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v35
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s79, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v31
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v32
+; SI-NEXT: s_and_b32 s4, s29, 0xff
+; SI-NEXT: s_lshl_b32 s6, s78, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v29
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s77, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v23
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s76, 24
+; SI-NEXT: v_or_b32_e32 v4, v6, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v25
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v19
+; SI-NEXT: s_and_b32 s4, s45, 0xff
+; SI-NEXT: s_lshl_b32 s6, s75, 8
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v16
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_and_b32 s6, s74, 0xff
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_lshl_b32 s6, s6, 16
+; SI-NEXT: s_lshl_b32 s8, s73, 24
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_and_b32 s4, s4, 0xffff
+; SI-NEXT: s_or_b32 s6, s8, s6
+; SI-NEXT: s_or_b32 s4, s4, s6
+; SI-NEXT: s_lshl_b32 s6, s72, 8
+; SI-NEXT: s_lshl_b32 s8, s62, 24
+; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: v_readlane_b32 s30, v63, 34
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s99, v63, 33
@@ -111091,378 +110863,239 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:12
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:60
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v41
-; SI-NEXT: s_and_b32 s4, s21, 0xff
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v18
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s43, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, vcc_hi, 0xff
+; SI-NEXT: s_and_b32 s6, s63, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s95, 8
-; SI-NEXT: s_lshl_b32 s8, s93, 24
+; SI-NEXT: s_lshl_b32 s6, s61, 8
+; SI-NEXT: s_lshl_b32 s8, s59, 24
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:16
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:20
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v50
-; SI-NEXT: s_and_b32 s4, s23, 0xff
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s94, 0xff
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s92, 8
-; SI-NEXT: s_lshl_b32 s8, s90, 24
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:24
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:28
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v22
-; SI-NEXT: s_and_b32 s4, s25, 0xff
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v13
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s41, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s91, 0xff
+; SI-NEXT: s_and_b32 s6, s60, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s89, 8
-; SI-NEXT: s_lshl_b32 s8, s79, 24
+; SI-NEXT: s_lshl_b32 s6, s58, 8
+; SI-NEXT: s_lshl_b32 s8, s56, 24
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:32
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v16
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v21
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_and_b32 s4, s27, 0xff
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v11
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s15, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s88, 0xff
+; SI-NEXT: s_and_b32 s6, s57, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s78, 8
-; SI-NEXT: s_lshl_b32 s8, s76, 24
+; SI-NEXT: s_lshl_b32 s6, s47, 8
+; SI-NEXT: s_lshl_b32 s8, s44, 24
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v6
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; SI-NEXT: v_or_b32_e32 v2, v2, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v13
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v14
-; SI-NEXT: s_and_b32 s4, s29, 0xff
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v18
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s77, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v3
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:80
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:52
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v36
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_and_b32 s4, s45, 0xff
-; SI-NEXT: s_lshl_b32 s6, s75, 8
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v9
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s13, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s74, 0xff
+; SI-NEXT: s_and_b32 s6, s46, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s73, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s72, 8
-; SI-NEXT: s_lshl_b32 s8, s62, 24
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v3
+; SI-NEXT: s_lshl_b32 s6, s42, 8
+; SI-NEXT: s_lshl_b32 s8, s28, 24
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v5
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v33
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v3
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:60
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v12
-; SI-NEXT: s_and_b32 s4, s43, 0xff
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v48
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s63, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v23
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:88
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v47
-; SI-NEXT: s_and_b32 s4, s41, 0xff
-; SI-NEXT: s_lshl_b32 s6, s61, 8
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v7
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s11, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s60, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v59
+; SI-NEXT: s_and_b32 s6, s40, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s59, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s58, 8
-; SI-NEXT: s_lshl_b32 s8, s56, 24
-; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v46
+; SI-NEXT: s_lshl_b32 s6, s26, 8
+; SI-NEXT: s_lshl_b32 s8, s22, 24
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v12
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v45
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v6
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:72
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v52
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v53
-; SI-NEXT: s_and_b32 s4, s15, 0xff
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v26
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s57, 0xff
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v35
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:80
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:96
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
+; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v44
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v39
-; SI-NEXT: s_and_b32 s4, s13, 0xff
-; SI-NEXT: s_lshl_b32 s6, s47, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v55
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_and_b32 s4, s9, 0xff
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s46, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v58
+; SI-NEXT: s_and_b32 s6, s24, 0xff
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s44, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:88
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v28
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v1
-; SI-NEXT: s_and_b32 s4, s11, 0xff
-; SI-NEXT: s_lshl_b32 s6, s42, 8
+; SI-NEXT: s_lshl_b32 s6, s20, 8
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v56
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s40, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v30
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s28, 24
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 24, v5
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; SI-NEXT: v_or_b32_e32 v4, v6, v4
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:96
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v4, 8, v8
-; SI-NEXT: s_and_b32 s4, s9, 0xff
-; SI-NEXT: s_lshl_b32 s6, s26, 8
-; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xff, v20
-; SI-NEXT: s_or_b32 s4, s4, s6
-; SI-NEXT: s_and_b32 s6, s24, 0xff
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s8, s22, 24
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v4, v5, v4
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_or_b32 s6, s8, s6
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:104
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s7, 0xff
-; SI-NEXT: s_lshl_b32 s6, s20, 8
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_and_b32 s6, s18, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v54
; SI-NEXT: s_lshl_b32 s6, s6, 16
; SI-NEXT: s_lshl_b32 s7, s16, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: s_lshl_b32 s6, s10, 24
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v1
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 8, v3
; SI-NEXT: v_or_b32_e32 v2, v2, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xff, v19
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 24, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:112
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
-; SI-NEXT: v_and_b32_e32 v1, 0xff, v51
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v34
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: s_and_b32 s4, s5, 0xff
; SI-NEXT: s_lshl_b32 s5, s14, 8
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v15
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: s_and_b32 s5, s12, 0xff
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v38
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_or_b32_e32 v2, v3, v2
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_or_b32_e32 v2, v3, v2
+; SI-NEXT: v_or_b32_e32 v1, v1, v2
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
@@ -112350,28 +111983,27 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; VI-NEXT: .LBB73_6: ; %end
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
-; VI-NEXT: v_perm_b32 v55, v58, v46, s4
-; VI-NEXT: v_perm_b32 v49, v49, v45, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v36, v36, v22, s4
-; VI-NEXT: v_perm_b32 v17, v38, v17, s4
+; VI-NEXT: v_perm_b32 v38, v38, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v22, v36, v22, s4
; VI-NEXT: v_perm_b32 v18, v32, v18, s4
; VI-NEXT: v_perm_b32 v28, v28, v52, s4
-; VI-NEXT: v_perm_b32 v35, v35, v54, s4
; VI-NEXT: v_perm_b32 v24, v24, v25, s4
-; VI-NEXT: v_perm_b32 v50, v50, v59, s4
-; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v29, v44, v29, s4
; VI-NEXT: v_perm_b32 v33, v51, v33, s4
+; VI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v35, v35, v54, s4
+; VI-NEXT: v_perm_b32 v16, v16, v53, s4
+; VI-NEXT: v_perm_b32 v29, v44, v29, s4
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; VI-NEXT: v_or_b32_e32 v29, v33, v29
; VI-NEXT: v_perm_b32 v48, v48, v57, s4
+; VI-NEXT: v_perm_b32 v21, v41, v21, s4
+; VI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; VI-NEXT: v_or_b32_e32 v21, v22, v21
+; VI-NEXT: v_perm_b32 v49, v49, v45, s4
; VI-NEXT: v_perm_b32 v37, v37, v56, s4
-; VI-NEXT: v_perm_b32 v16, v16, v53, s4
-; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v55
-; VI-NEXT: v_or_b32_e32 v50, v50, v53
+; VI-NEXT: v_perm_b32 v55, v58, v46, s4
+; VI-NEXT: v_perm_b32 v50, v50, v59, s4
; VI-NEXT: v_perm_b32 v20, v20, v47, s4
; VI-NEXT: v_readlane_b32 s30, v63, 30
; VI-NEXT: v_readlane_b32 s31, v63, 31
@@ -112405,113 +112037,103 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_readlane_b32 s36, v63, 2
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
-; VI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_perm_b32 v39, v39, v45, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(4)
-; VI-NEXT: v_perm_b32 v51, v40, v58, s4
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v25, v15, v25, s4
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v26, v26, v45, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v38, v22, v45, s4
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v31, v31, v22, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v32, v22, v45, s4
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v27, v27, v22, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v52, v22, v45, s4
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v54, v23, v22, s4
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v22, v23, v22, s4
-; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v19, v19, v23, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v23, v23, v45, s4
-; VI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v26, v26, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; VI-NEXT: v_or_b32_e32 v26, v35, v26
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v36, v36, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v31, v31, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v32, v32, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v27, v27, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v52, v52, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v23, v23, v17, s4
+; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v17, v25, v17, s4
+; VI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v51, v40, v51, s4
+; VI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v19, v19, v25, s4
+; VI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; VI-NEXT: v_or_b32_e32 v17, v19, v17
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v25, v54, v25, s4
+; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v39, v39, v40, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v54, v15, v54, s4
+; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:4
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v51
; VI-NEXT: v_or_b32_e32 v29, v48, v29
; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:8
; VI-NEXT: v_perm_b32 v29, v43, v61, s4
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; VI-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:28
+; VI-NEXT: v_lshlrev_b32_e32 v21, 16, v36
; VI-NEXT: v_or_b32_e32 v29, v49, v29
+; VI-NEXT: v_or_b32_e32 v21, v31, v21
; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:12
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v39
+; VI-NEXT: buffer_store_dword v21, v0, s[0:3], 0 offen offset:32
+; VI-NEXT: v_perm_b32 v21, v34, v30, s4
; VI-NEXT: v_or_b32_e32 v29, v37, v29
+; VI-NEXT: v_lshlrev_b32_e32 v21, 16, v21
; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:16
; VI-NEXT: v_perm_b32 v29, v60, v42, s4
+; VI-NEXT: v_or_b32_e32 v18, v18, v21
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; VI-NEXT: v_or_b32_e32 v17, v17, v29
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:20
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v26
-; VI-NEXT: v_or_b32_e32 v17, v35, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:24
-; VI-NEXT: v_perm_b32 v17, v41, v21, s4
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; VI-NEXT: v_or_b32_e32 v17, v36, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:28
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v38
-; VI-NEXT: v_or_b32_e32 v17, v31, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:32
-; VI-NEXT: v_perm_b32 v17, v34, v30, s4
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; VI-NEXT: v_or_b32_e32 v17, v18, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:36
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; VI-NEXT: v_or_b32_e32 v17, v27, v17
+; VI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:36
+; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v32
+; VI-NEXT: v_or_b32_e32 v29, v38, v29
+; VI-NEXT: v_or_b32_e32 v18, v27, v18
+; VI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:20
+; VI-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:24
+; VI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:40
+; VI-NEXT: s_waitcnt vmcnt(10)
+; VI-NEXT: v_perm_b32 v15, v53, v15, s4
+; VI-NEXT: v_lshlrev_b32_e32 v53, 16, v55
+; VI-NEXT: v_or_b32_e32 v50, v50, v53
; VI-NEXT: buffer_store_dword v50, v0, s[0:3], 0 offen
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:40
-; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(13)
-; VI-NEXT: v_perm_b32 v15, v15, v45, s4
+; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v17, v18, v17, s4
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; VI-NEXT: v_or_b32_e32 v17, v28, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:44
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v52
-; VI-NEXT: v_or_b32_e32 v17, v54, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:48
-; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v17, v18, v17, s4
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; VI-NEXT: v_or_b32_e32 v17, v24, v17
-; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:52
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v22
-; VI-NEXT: v_or_b32_e32 v17, v19, v17
+; VI-NEXT: v_perm_b32 v18, v21, v18, s4
+; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT: v_or_b32_e32 v18, v28, v18
+; VI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:44
+; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v52
+; VI-NEXT: v_or_b32_e32 v18, v23, v18
+; VI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:48
+; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:56
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v18, v21, v18, s4
+; VI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; VI-NEXT: v_or_b32_e32 v18, v24, v18
+; VI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:52
; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -112519,8 +112141,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; VI-NEXT: v_or_b32_e32 v17, v20, v17
; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:60
-; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v23
-; VI-NEXT: v_or_b32_e32 v17, v25, v17
+; VI-NEXT: v_lshlrev_b32_e32 v17, 16, v25
+; VI-NEXT: v_or_b32_e32 v17, v54, v17
; VI-NEXT: buffer_store_dword v17, v0, s[0:3], 0 offen offset:64
; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
@@ -112549,8 +112171,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_perm_b32 v9, v9, v13, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v13, v13, v14, s4
; VI-NEXT: v_lshlrev_b32_e32 v13, 16, v13
; VI-NEXT: v_or_b32_e32 v9, v9, v13
@@ -112569,10 +112190,9 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v7, v7, v9, s4
; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v9, v9, v13, s4
+; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v9, v9, v10, s4
; VI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-NEXT: v_or_b32_e32 v7, v7, v9
; VI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:88
@@ -112591,8 +112211,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_perm_b32 v5, v5, v7, s4
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v7, v7, v8, s4
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: v_or_b32_e32 v5, v5, v7
@@ -112612,8 +112231,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_perm_b32 v3, v3, v5, s4
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v5, v5, v6, s4
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_or_b32_e32 v3, v3, v5
@@ -112633,8 +112251,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_perm_b32 v1, v1, v3, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v3, v3, v4, s4
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -112649,12 +112266,12 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v11, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -112694,8 +112311,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
@@ -113117,46 +112734,46 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[17:18]
; GFX9-NEXT: v_add_f64 v[48:49], s[16:17], 1.0
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[21:22]
; GFX9-NEXT: v_lshrrev_b32_e32 v57, 16, v11
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[23:24]
-; GFX9-NEXT: v_lshrrev_b32_e32 v55, 8, v22
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v13
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[28:29]
-; GFX9-NEXT: v_lshrrev_b32_e32 v58, 8, v24
+; GFX9-NEXT: v_lshrrev_b32_e32 v55, 8, v22
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[32:33]
-; GFX9-NEXT: v_lshrrev_b32_e32 v50, 8, v29
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v58, 8, v24
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[34:35]
-; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v28
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v50, 8, v29
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[36:37]
-; GFX9-NEXT: v_lshrrev_b32_e32 v31, 16, v33
+; GFX9-NEXT: v_lshrrev_b32_e32 v31, 24, v33
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[38:39]
-; GFX9-NEXT: v_lshrrev_b32_e32 v54, 24, v35
+; GFX9-NEXT: v_lshrrev_b32_e32 v53, 16, v33
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[48:49]
-; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v35
+; GFX9-NEXT: v_lshrrev_b32_e32 v54, 24, v35
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
@@ -113224,53 +112841,51 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v14
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v13
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v13
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v18
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v18
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v18
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v22
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v22
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v21
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v21
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v24
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v24
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v23
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v23
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v29
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v29
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v28
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v28
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v33
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v32
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v32
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v34
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 8, v33
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v47, 16, v35
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 8, v35
; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v34
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v34
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 24, v37
; GFX9-NEXT: v_lshrrev_b32_e32 v41, 16, v37
; GFX9-NEXT: v_lshrrev_b32_e32 v44, 8, v37
@@ -113288,28 +112903,26 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_lshrrev_b32_e32 v60, 8, v48
; GFX9-NEXT: s_branch .LBB73_6
; GFX9-NEXT: .LBB73_5:
-; GFX9-NEXT: v_mov_b32_e32 v15, s66
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s65
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s64
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v15, s55
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s54
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s53
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s52
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s51
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s50
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s49
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s48
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v15, s39
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s38
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s99
@@ -113404,27 +113017,31 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 24
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 25
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 26
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 27
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 28
; GFX9-NEXT: v_mov_b32_e32 v55, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 29
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 30
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 31
; GFX9-NEXT: v_mov_b32_e32 v58, s4
+; GFX9-NEXT: v_mov_b32_e32 v57, s34
+; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v57, s30
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -113434,34 +113051,31 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v57, s94
-; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v57, s92
-; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v52, s34
-; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v57, s90
; GFX9-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v52, s66
; GFX9-NEXT: v_mov_b32_e32 v57, v51
; GFX9-NEXT: v_mov_b32_e32 v51, s88
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v51, s78
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v51, s76
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v51, s74
; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -113487,7 +113101,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; GFX9-NEXT: v_readlane_b32 s4, v62, 32
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 33
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
@@ -113495,10 +113109,9 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readlane_b32 s4, v62, 34
; GFX9-NEXT: v_mov_b32_e32 v50, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 35
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v15, s4
-; GFX9-NEXT: v_readlane_b32 s4, v62, 36
; GFX9-NEXT: v_mov_b32_e32 v31, s4
+; GFX9-NEXT: v_readlane_b32 s4, v62, 36
+; GFX9-NEXT: v_mov_b32_e32 v53, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 37
; GFX9-NEXT: v_mov_b32_e32 v16, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 38
@@ -113518,7 +113131,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readlane_b32 s4, v62, 45
; GFX9-NEXT: v_mov_b32_e32 v26, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 46
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v15, s4
; GFX9-NEXT: v_readlane_b32 s4, v62, 47
; GFX9-NEXT: v_mov_b32_e32 v61, s4
@@ -113568,28 +113181,22 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_mov_b32_e32 v19, s68
; GFX9-NEXT: v_mov_b32_e32 v27, s67
; GFX9-NEXT: v_mov_b32_e32 v20, s4
-; GFX9-NEXT: v_mov_b32_e32 v51, s55
+; GFX9-NEXT: v_mov_b32_e32 v51, s39
; GFX9-NEXT: .LBB73_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v39, v39, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v49, v49, v20, s4
-; GFX9-NEXT: v_perm_b32 v36, v36, v19, s4
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v38, v40, s4
; GFX9-NEXT: v_perm_b32 v30, v35, v30, s4
+; GFX9-NEXT: v_perm_b32 v20, v49, v20, s4
+; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v16, v33, v16, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v19, v36, v19, s4
+; GFX9-NEXT: v_perm_b32 v36, v37, v44, s4
+; GFX9-NEXT: v_perm_b32 v29, v29, v50, s4
+; GFX9-NEXT: v_perm_b32 v34, v34, v52, s4
+; GFX9-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v48, v48, v60, s4
-; GFX9-NEXT: v_perm_b32 v25, v41, v25, s4
-; GFX9-NEXT: v_perm_b32 v37, v37, v44, s4
-; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX9-NEXT: v_or_b32_e32 v25, v37, v25
-; GFX9-NEXT: v_perm_b32 v26, v26, v42, s4
-; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX9-NEXT: v_perm_b32 v38, v38, v40, s4
-; GFX9-NEXT: v_or_b32_e32 v26, v39, v26
; GFX9-NEXT: v_perm_b32 v24, v24, v58, s4
; GFX9-NEXT: v_perm_b32 v22, v22, v55, s4
; GFX9-NEXT: v_readlane_b32 s30, v63, 34
@@ -113628,138 +113235,129 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v34, v34, v15, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v27, v27, v19, s4
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v45, v59, v52, s4
-; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v35, v15, v19, s4
+; GFX9-NEXT: v_perm_b32 v40, v46, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v27, v27, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v35, v35, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v56, v56, v59, s4
-; GFX9-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v19, v51, v19, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v32, v32, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v33, v33, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v40, v46, v59, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v20, v28, v15, s4
-; GFX9-NEXT: v_perm_b32 v28, v29, v50, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v28, v28, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v29, v15, v50, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v37, v37, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v23, v23, v15, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v33, v15, v50, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v50, v50, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v21, v21, v15, s4
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v50, v15, v50, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v52, v52, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v45, v59, v45, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v17, v17, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v18, v18, v15, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v49, v56, v49, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
+; GFX9-NEXT: v_perm_b32 v15, v51, v15, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v51, 16, v45
-; GFX9-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:20
-; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v27
; GFX9-NEXT: v_or_b32_e32 v48, v48, v51
-; GFX9-NEXT: v_or_b32_e32 v25, v34, v25
; GFX9-NEXT: buffer_store_dword v48, v0, s[0:3], 0 offen
; GFX9-NEXT: v_perm_b32 v48, v43, v61, s4
-; GFX9-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:24
-; GFX9-NEXT: v_perm_b32 v25, v47, v54, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v48
-; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX9-NEXT: v_or_b32_e32 v48, v49, v48
-; GFX9-NEXT: v_or_b32_e32 v25, v30, v25
-; GFX9-NEXT: buffer_store_dword v48, v0, s[0:3], 0 offen offset:4
-; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v56
-; GFX9-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:12
-; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v40
-; GFX9-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:28
-; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v35
-; GFX9-NEXT: v_or_b32_e32 v38, v38, v48
-; GFX9-NEXT: v_or_b32_e32 v26, v36, v26
-; GFX9-NEXT: v_or_b32_e32 v25, v32, v25
-; GFX9-NEXT: buffer_store_dword v38, v0, s[0:3], 0 offen offset:8
-; GFX9-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:16
-; GFX9-NEXT: buffer_store_dword v25, v0, s[0:3], 0 offen offset:32
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v25, v31, v25, s4
-; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX9-NEXT: v_or_b32_e32 v16, v16, v25
+; GFX9-NEXT: v_or_b32_e32 v20, v20, v48
+; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v49
+; GFX9-NEXT: v_or_b32_e32 v20, v38, v20
+; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:8
+; GFX9-NEXT: v_perm_b32 v20, v26, v42, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX9-NEXT: v_or_b32_e32 v20, v39, v20
+; GFX9-NEXT: buffer_store_dword v20, v0, s[0:3], 0 offen offset:12
+; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v40
+; GFX9-NEXT: v_or_b32_e32 v19, v19, v20
+; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:16
+; GFX9-NEXT: v_perm_b32 v19, v41, v25, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_or_b32_e32 v19, v36, v19
+; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:20
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v27
+; GFX9-NEXT: v_or_b32_e32 v19, v34, v19
+; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:24
+; GFX9-NEXT: v_perm_b32 v19, v47, v54, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_or_b32_e32 v19, v30, v19
+; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:28
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v35
+; GFX9-NEXT: v_or_b32_e32 v19, v32, v19
+; GFX9-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:32
+; GFX9-NEXT: v_perm_b32 v19, v53, v31, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX9-NEXT: v_or_b32_e32 v16, v16, v19
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:36
-; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v19
-; GFX9-NEXT: v_or_b32_e32 v16, v20, v16
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v33
+; GFX9-NEXT: v_or_b32_e32 v16, v28, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:40
; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v16, v19, v16, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX9-NEXT: v_or_b32_e32 v16, v28, v16
+; GFX9-NEXT: v_or_b32_e32 v16, v29, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:44
-; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v29
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v37
; GFX9-NEXT: v_or_b32_e32 v16, v23, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:48
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v16, v19, v16, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX9-NEXT: v_or_b32_e32 v16, v24, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:52
-; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v33
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v50
; GFX9-NEXT: v_or_b32_e32 v16, v21, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:56
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v16, v19, v16, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX9-NEXT: v_or_b32_e32 v16, v22, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:60
-; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v50
+; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v52
; GFX9-NEXT: v_or_b32_e32 v16, v17, v16
; GFX9-NEXT: buffer_store_dword v16, v0, s[0:3], 0 offen offset:64
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v16, v17, v16, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v16
@@ -113784,7 +113382,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v9, v13, s4
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v13, v14, v13, s4
@@ -113805,7 +113402,6 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v7, v9, s4
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v10, v9, s4
@@ -113827,8 +113423,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_perm_b32 v5, v5, v7, s4
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v7, v8, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX9-NEXT: v_or_b32_e32 v5, v5, v7
@@ -113848,8 +113443,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_perm_b32 v3, v3, v5, s4
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v5, v6, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
@@ -113869,8 +113463,7 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v3, v4, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
@@ -113888,10 +113481,10 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v2, v57, v2, s4
; GFX9-NEXT: v_perm_b32 v1, v11, v1, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v2, v57, v2, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -113920,8 +113513,8 @@ define inreg <128 x i8> @bitcast_v16f64_to_v128i8_scalar(<16 x double> inreg %a,
; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -129614,6 +129207,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v9
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v12
; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -129622,10 +129218,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v37, v36
; SI-NEXT: v_mov_b32_e32 v36, v47
; SI-NEXT: v_mov_b32_e32 v50, v49
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; SI-NEXT: v_mov_b32_e32 v40, v41
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v1, v43
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
@@ -129672,10 +129267,13 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v11
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v44, v45
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
; SI-NEXT: v_mov_b32_e32 v32, v59
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v9
@@ -129699,11 +129297,11 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v16
; SI-NEXT: s_waitcnt vmcnt(11)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v17
-; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v58, 16, v18
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v27
; SI-NEXT: s_waitcnt vmcnt(3)
@@ -129727,10 +129325,8 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v25
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v26
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[14:15], v[15:16], 16
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
@@ -129770,6 +129366,9 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v29
; SI-NEXT: v_mov_b32_e32 v39, v62
+; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
+; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v58, v56
; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -129777,38 +129376,27 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_mov_b32_e32 v21, v22
; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[21:22], v[22:23], 16
; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v55
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v22
; SI-NEXT: v_lshr_b64 v[22:23], v[37:38], 16
; SI-NEXT: v_lshr_b64 v[23:24], v[46:47], 16
; SI-NEXT: v_lshr_b64 v[24:25], v[48:49], 16
; SI-NEXT: v_lshr_b64 v[25:26], v[50:51], 16
; SI-NEXT: v_lshr_b64 v[26:27], v[44:45], 16
+; SI-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v28
; SI-NEXT: v_lshr_b64 v[27:28], v[42:43], 16
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v53
-; SI-NEXT: v_lshr_b64 v[61:62], v[52:53], 16
-; SI-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v45, v44
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v47, v36
; SI-NEXT: v_mov_b32_e32 v36, v37
; SI-NEXT: v_mov_b32_e32 v49, v50
-; SI-NEXT: v_mov_b32_e32 v51, v33
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_mov_b32_e32 v45, v44
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshr_b64 v[28:29], v[28:29], 16
; SI-NEXT: v_lshr_b64 v[29:30], v[40:41], 16
; SI-NEXT: v_lshr_b64 v[30:31], v[54:55], 16
@@ -129863,54 +129451,54 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: .LBB79_3: ; %Flow
; SI-NEXT: v_mov_b32_e32 v37, v32
; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v33, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v38, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v50, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v40, v56
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_cbranch_scc1 .LBB79_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v55
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v43
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: v_mov_b32_e32 v34, v56
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v19, 0xffff0000, v47
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v19
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
@@ -129918,176 +129506,152 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; SI-NEXT: v_lshr_b64 v[1:2], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v34
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(12)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
+; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_and_b32_e32 v23, 0xffff0000, v23
; SI-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v23
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
-; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(9)
; SI-NEXT: v_and_b32_e32 v24, 0xffff0000, v24
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff0000, v25
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_and_b32_e32 v26, 0xffff0000, v26
+; SI-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v27
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v28
+; SI-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v29, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v30
+; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; SI-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v62
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v6
; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v40
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v8
; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v9
; SI-NEXT: v_lshr_b64 v[8:9], v[8:9], 16
; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v10
; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v50
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
; SI-NEXT: v_lshr_b64 v[10:11], v[10:11], 16
; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v11
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v12
; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: v_lshr_b64 v[12:13], v[12:13], 16
; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; SI-NEXT: v_lshr_b64 v[13:14], v[13:14], 16
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v39
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
; SI-NEXT: v_lshr_b64 v[14:15], v[14:15], 16
; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; SI-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; SI-NEXT: v_lshr_b64 v[15:16], v[15:16], 16
+; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v32
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -130095,26 +129659,26 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; SI-NEXT: v_and_b32_e32 v31, 0xffff0000, v31
; SI-NEXT: v_and_b32_e32 v32, 0xffff0000, v52
+; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; SI-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v33
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v20
; SI-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v33
; SI-NEXT: v_lshr_b64 v[32:33], v[32:33], 16
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshr_b64 v[17:18], v[17:18], 16
; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v37
; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
@@ -130123,26 +129687,14 @@ define inreg <16 x double> @bitcast_v64bf16_to_v16f64_scalar(<64 x bfloat> inreg
; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
; SI-NEXT: v_lshr_b64 v[19:20], v[19:20], 16
; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v20
; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v21
; SI-NEXT: v_lshr_b64 v[20:21], v[20:21], 16
; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v21
; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; SI-NEXT: v_lshrrev_b32_e32 v22, 16, v22
; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v36
; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
@@ -153346,13 +152898,13 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v55, v55, v41, s4
; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; VI-NEXT: v_perm_b32 v37, v37, v42, s4
; VI-NEXT: v_perm_b32 v53, v59, v43, s4
; VI-NEXT: v_perm_b32 v1, v1, v60, s4
; VI-NEXT: v_perm_b32 v33, v33, v39, s4
; VI-NEXT: v_lshlrev_b32_e32 v39, 16, v53
; VI-NEXT: v_or_b32_e32 v1, v1, v39
; VI-NEXT: v_perm_b32 v2, v2, v58, s4
+; VI-NEXT: v_perm_b32 v37, v37, v42, s4
; VI-NEXT: v_perm_b32 v3, v3, v56, s4
; VI-NEXT: v_perm_b32 v4, v4, v45, s4
; VI-NEXT: v_perm_b32 v17, v17, v34, s4
@@ -153400,8 +152952,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v14, v14, v41, s4
; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v40, v40, v41, s4
; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -153503,9 +153054,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v1, v21, v36, s4
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v35, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -153521,13 +153071,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v23, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -153542,13 +153091,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v25, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -153563,13 +153111,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v27, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
@@ -153584,13 +153131,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v29, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v29, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
@@ -153605,13 +153151,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v1, v31, v1, s4
-; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v31, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
@@ -153663,10 +153208,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:4
; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:8
-; GFX9-NEXT: buffer_load_dword v60, off, s[0:3], s32
+; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32
; GFX9-NEXT: ; implicit-def: $vgpr33
; GFX9-NEXT: ; kill: killed $vgpr33
; GFX9-NEXT: ; implicit-def: $vgpr33
@@ -153736,12 +153280,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: ; implicit-def: $vgpr43
; GFX9-NEXT: ; kill: killed $vgpr35
; GFX9-NEXT: ; implicit-def: $vgpr35
-; GFX9-NEXT: v_mov_b32_e32 v56, v15
; GFX9-NEXT: ; kill: killed $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; kill: killed $vgpr33
; GFX9-NEXT: ; implicit-def: $vgpr33
-; GFX9-NEXT: ; implicit-def: $vgpr15
; GFX9-NEXT: ; kill: killed $vgpr43
; GFX9-NEXT: ; implicit-def: $vgpr43
; GFX9-NEXT: ; kill: killed $vgpr35
@@ -153750,20 +153292,18 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; kill: killed $vgpr33
; GFX9-NEXT: ; implicit-def: $vgpr33
-; GFX9-NEXT: ; kill: killed $vgpr15
-; GFX9-NEXT: ; implicit-def: $vgpr15
; GFX9-NEXT: ; kill: killed $vgpr43
; GFX9-NEXT: ; kill: killed $vgpr35
; GFX9-NEXT: ; implicit-def: $vgpr43
; GFX9-NEXT: ; implicit-def: $vgpr35
-; GFX9-NEXT: v_mov_b32_e32 v57, v16
+; GFX9-NEXT: v_mov_b32_e32 v41, v16
+; GFX9-NEXT: v_mov_b32_e32 v40, v15
; GFX9-NEXT: ; kill: killed $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr16
; GFX9-NEXT: ; implicit-def: $vgpr44
; GFX9-NEXT: ; kill: killed $vgpr33
; GFX9-NEXT: ; implicit-def: $vgpr33
-; GFX9-NEXT: ; kill: killed $vgpr15
; GFX9-NEXT: ; implicit-def: $vgpr34
; GFX9-NEXT: ; implicit-def: $vgpr36
; GFX9-NEXT: ; kill: killed $vgpr43
@@ -153771,41 +153311,53 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: ; implicit-def: $vgpr43
; GFX9-NEXT: ; implicit-def: $vgpr35
; GFX9-NEXT: ; implicit-def: $vgpr15
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr55
-; GFX9-NEXT: ; implicit-def: $vgpr40
+; GFX9-NEXT: ; implicit-def: $vgpr63
; GFX9-NEXT: ; implicit-def: $vgpr54
-; GFX9-NEXT: ; implicit-def: $vgpr32
+; GFX9-NEXT: ; implicit-def: $vgpr56
; GFX9-NEXT: ; kill: killed $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: ; implicit-def: $vgpr49
-; GFX9-NEXT: ; implicit-def: $vgpr53
+; GFX9-NEXT: ; implicit-def: $vgpr32
; GFX9-NEXT: ; implicit-def: $vgpr47
; GFX9-NEXT: ; implicit-def: $vgpr59
-; GFX9-NEXT: ; implicit-def: $vgpr42
+; GFX9-NEXT: ; implicit-def: $vgpr53
; GFX9-NEXT: ; implicit-def: $vgpr37
; GFX9-NEXT: ; implicit-def: $vgpr48
; GFX9-NEXT: ; implicit-def: $vgpr38
-; GFX9-NEXT: ; implicit-def: $vgpr63
-; GFX9-NEXT: ; implicit-def: $vgpr41
+; GFX9-NEXT: ; implicit-def: $vgpr61
+; GFX9-NEXT: ; implicit-def: $vgpr60
; GFX9-NEXT: ; implicit-def: $vgpr62
; GFX9-NEXT: ; implicit-def: $vgpr39
-; GFX9-NEXT: ; implicit-def: $vgpr52
+; GFX9-NEXT: ; implicit-def: $vgpr57
+; GFX9-NEXT: ; implicit-def: $vgpr42
; GFX9-NEXT: ; kill: killed $vgpr33
; GFX9-NEXT: ; implicit-def: $vgpr33
+; GFX9-NEXT: ; implicit-def: $vgpr52
; GFX9-NEXT: ; kill: killed $vgpr34
; GFX9-NEXT: ; implicit-def: $vgpr34
; GFX9-NEXT: ; implicit-def: $vgpr46
; GFX9-NEXT: ; kill: killed $vgpr36
; GFX9-NEXT: ; kill: killed $vgpr43
; GFX9-NEXT: ; kill: killed $vgpr35
-; GFX9-NEXT: ; implicit-def: $vgpr50
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr15
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-NEXT: ; implicit-def: $vgpr15
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-NEXT: ; implicit-def: $vgpr15
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr15
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -153818,22 +153370,22 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-NEXT: ; implicit-def: $vgpr15
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr35
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr15
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr15
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: ; implicit-def: $vgpr35
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr35
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
@@ -153846,16 +153398,12 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-NEXT: ; implicit-def: $vgpr35
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; GFX9-NEXT: ; implicit-def: $vgpr58
; GFX9-NEXT: ; implicit-def: $vgpr43
-; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(29)
+; GFX9-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(33)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
; GFX9-NEXT: ; implicit-def: $vgpr31
; GFX9-NEXT: ; kill: killed $vgpr31
@@ -153870,202 +153418,206 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB90_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[56:57]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v57
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[13:14]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[40:41]
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[11:12]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v57
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v41
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v41
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v41
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v40
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v3
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v40
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v19
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v36, 24, v10
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v1
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v51
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v51
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v51
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GFX9-NEXT: s_waitcnt vmcnt(51)
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v50
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v50
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v30
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v30
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v29
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v29
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v28
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v28
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v24
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v28
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v23
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v27
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v22
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v26
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v21
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v26
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v20
+; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v25
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v18
+; GFX9-NEXT: v_lshrrev_b32_e32 v47, 8, v10
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 8, v9
+; GFX9-NEXT: v_lshrrev_b32_e32 v31, 8, v8
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 8, v7
+; GFX9-NEXT: v_lshrrev_b32_e32 v49, 8, v6
+; GFX9-NEXT: v_lshrrev_b32_e32 v56, 8, v4
+; GFX9-NEXT: s_waitcnt vmcnt(35)
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[13:14]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v14
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[9:10]
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v4
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[11:12]
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[7:8]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[9:10]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v14
; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[7:8]
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[5:6]
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v57
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v14
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[3:4]
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v4
-; GFX9-NEXT: s_waitcnt vmcnt(45)
-; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[60:61]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v56
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v3
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v56
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v19
-; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[1:2]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v13
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v14
-; GFX9-NEXT: v_lshrrev_b32_e32 v36, 24, v10
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v1
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v35, 16, v17
+; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[50:51]
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[29:30]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v14
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v61
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: v_mov_b32_e32 v59, v36
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v14
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v61
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v13
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v61
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v13
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v60
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[25:26]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v12
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v60
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v12
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v30
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v12
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v30
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[23:24]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v30
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v11
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v29
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v29
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[21:22]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v28
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v8
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v28
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v24
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v28
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v23
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[19:20]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v27
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v22
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v26
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v21
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v6
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v26
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v25
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v20
; GFX9-NEXT: v_lshrrev_b64 v[35:36], 24, v[17:18]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v25
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 8, v24
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 16, v18
-; GFX9-NEXT: v_lshrrev_b32_e32 v47, 8, v10
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 8, v9
-; GFX9-NEXT: v_lshrrev_b32_e32 v31, 8, v8
-; GFX9-NEXT: v_lshrrev_b32_e32 v53, 8, v7
-; GFX9-NEXT: v_lshrrev_b32_e32 v49, 8, v6
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v5
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 8, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 8, v3
-; GFX9-NEXT: v_lshrrev_b32_e32 v40, 8, v2
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v63, 8, v2
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 8, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 8, v27
; GFX9-NEXT: v_lshrrev_b32_e32 v34, 8, v26
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v25
; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v24
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v23
+; GFX9-NEXT: v_lshrrev_b32_e32 v42, 8, v24
+; GFX9-NEXT: v_lshrrev_b32_e32 v57, 8, v23
; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v22
; GFX9-NEXT: v_lshrrev_b32_e32 v44, 8, v22
; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v21
-; GFX9-NEXT: v_lshrrev_b32_e32 v41, 24, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v63, 8, v20
+; GFX9-NEXT: v_lshrrev_b32_e32 v60, 24, v20
+; GFX9-NEXT: v_lshrrev_b32_e32 v61, 8, v20
; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v19
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v18
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v18
-; GFX9-NEXT: v_lshrrev_b32_e32 v42, 8, v17
-; GFX9-NEXT: v_lshrrev_b64 v[50:51], 24, v[1:2]
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v53, 8, v17
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-NEXT: .LBB90_2: ; %Flow
; GFX9-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB90_4
@@ -154080,7 +153632,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v33, 0x400000, v31
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_bfe_u32 v31, v18, 16, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v32, v33, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v32, v33, vcc
; GFX9-NEXT: v_add3_u32 v31, v31, v18, s6
; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
@@ -154095,51 +153647,51 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v18
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v18, v18
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v0, v15, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v16, v0, v13, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v31, v32, vcc
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v31, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v20
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v31, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v31, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v31, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v20
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v62, v18, v31, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v15, v13, v0, s7
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v31, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v19
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v20, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v19
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v20, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v38, v18, v20, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v22
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v0, v62, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v16, v0, v13, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154154,9 +153706,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v21
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v0, v15, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v15, v0, v38, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154170,11 +153721,11 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v24
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-NEXT: v_perm_b32 v16, v0, v43, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
@@ -154183,7 +153734,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v24
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
; GFX9-NEXT: v_perm_b32 v15, v0, v44, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
@@ -154191,19 +153742,17 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v23
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v23
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v62, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154211,17 +153760,16 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v26
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v26
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v61, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154229,17 +153777,19 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v25
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_mov_b32_e32 v51, v16
-; GFX9-NEXT: v_mov_b32_e32 v50, v15
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v25
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v16, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154247,36 +153797,36 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v28
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v28
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v15, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v27
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v27
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v16, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
@@ -154284,8 +153834,8 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v30
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v13, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
@@ -154306,37 +153856,36 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v29
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: v_cndmask_b32_e32 v39, v18, v19, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v37, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v61
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v51
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; GFX9-NEXT: v_cndmask_b32_e32 v55, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v61
+; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v51
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v15, v0, s7
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v15, v13, v0, s7
; GFX9-NEXT: v_cndmask_b32_e32 v0, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: s_waitcnt vmcnt(57)
-; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v60
+; GFX9-NEXT: s_waitcnt vmcnt(60)
+; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v50
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; GFX9-NEXT: v_cndmask_b32_e32 v53, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
; GFX9-NEXT: v_add3_u32 v18, v18, v17, s6
; GFX9-NEXT: v_or_b32_e32 v19, 0x400000, v17
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v17, v17
-; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v60
+; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v50
; GFX9-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; GFX9-NEXT: v_cndmask_b32_e32 v30, v18, v19, vcc
; GFX9-NEXT: v_bfe_u32 v18, v17, 16, 1
@@ -154387,21 +153936,45 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX9-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v14, v54, v42, s7
+; GFX9-NEXT: v_perm_b32 v13, v55, v37, s7
; GFX9-NEXT: v_cndmask_b32_e32 v23, v2, v4, vcc
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; GFX9-NEXT: v_add3_u32 v2, v2, v1, s6
; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
+; GFX9-NEXT: v_perm_b32 v14, v53, v0, s7
+; GFX9-NEXT: v_perm_b32 v13, v52, v30, s7
; GFX9-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; GFX9-NEXT: v_cndmask_b32_e32 v24, v2, v4, vcc
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT: v_perm_b32 v14, v27, v29, s7
+; GFX9-NEXT: v_perm_b32 v13, v26, v28, s7
; GFX9-NEXT: v_add3_u32 v2, v2, v1, s6
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GFX9-NEXT: v_cndmask_b32_e32 v22, v2, v3, vcc
+; GFX9-NEXT: v_perm_b32 v14, v23, v25, s7
+; GFX9-NEXT: v_perm_b32 v13, v22, v24, s7
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v22, v2, v3, vcc
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add3_u32 v2, v2, v1, s6
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
@@ -154512,6 +154085,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v32, 0x400000, v11
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v11, v11
; GFX9-NEXT: v_cndmask_b32_e32 v11, v31, v32, vcc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v14
; GFX9-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
@@ -154529,21 +154103,13 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v13
; GFX9-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; GFX9-NEXT: v_mov_b32_e32 v34, v16
-; GFX9-NEXT: v_mov_b32_e32 v33, v15
-; GFX9-NEXT: v_perm_b32 v16, v54, v42, s7
-; GFX9-NEXT: v_perm_b32 v15, v55, v39, s7
; GFX9-NEXT: v_bfe_u32 v48, v31, 16, 1
; GFX9-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
; GFX9-NEXT: v_add3_u32 v48, v48, v31, s6
; GFX9-NEXT: v_or_b32_e32 v49, 0x400000, v31
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v31, v31
; GFX9-NEXT: v_bfe_u32 v31, v13, 16, 1
-; GFX9-NEXT: v_perm_b32 v16, v53, v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v61, v0
+; GFX9-NEXT: v_mov_b32_e32 v51, v0
; GFX9-NEXT: v_cndmask_b32_e32 v0, v48, v49, vcc
; GFX9-NEXT: v_add3_u32 v31, v31, v13, s6
; GFX9-NEXT: v_or_b32_e32 v48, 0x400000, v13
@@ -154552,68 +154118,69 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_cndmask_b32_e32 v13, v31, v48, vcc
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v14
; GFX9-NEXT: v_perm_b32 v31, v13, v0, s7
-; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v13
; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v12
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v18
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v23
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v22
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v27
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v26
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v53
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v52
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v54
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v55
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v15, v52, v30, s7
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v15, v26, v28, s7
-; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v28
-; GFX9-NEXT: v_perm_b32 v16, v27, v29, s7
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v16, v23, v25, s7
-; GFX9-NEXT: v_perm_b32 v15, v22, v24, s7
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v15, v56
-; GFX9-NEXT: v_mov_b32_e32 v16, v57
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v56, v9, v4, s7
+; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v28
; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v4
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v25
; GFX9-NEXT: v_perm_b32 v59, v12, v3, s7
; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v3
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v24
-; GFX9-NEXT: v_perm_b32 v38, v19, v21, s7
+; GFX9-NEXT: v_perm_b32 v35, v18, v20, s7
+; GFX9-NEXT: v_perm_b32 v57, v10, v5, s7
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v20
+; GFX9-NEXT: v_perm_b32 v36, v19, v21, s7
+; GFX9-NEXT: v_perm_b32 v58, v11, v2, s7
+; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v29
+; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v37
+; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v38
+; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: v_mov_b32_e32 v34, v16
+; GFX9-NEXT: v_mov_b32_e32 v33, v15
+; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; GFX9-NEXT: v_mov_b32_e32 v16, v41
; GFX9-NEXT: v_lshlrev_b32_e32 v48, 16, v16
; GFX9-NEXT: v_add_f32_e32 v48, 0x40c00000, v48
; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
@@ -154623,6 +154190,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v63, 0x400000, v48
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v48, v48
; GFX9-NEXT: v_bfe_u32 v48, v16, 16, 1
+; GFX9-NEXT: v_mov_b32_e32 v15, v40
; GFX9-NEXT: v_cndmask_b32_e32 v63, v49, v63, vcc
; GFX9-NEXT: v_add3_u32 v48, v48, v16, s6
; GFX9-NEXT: v_or_b32_e32 v49, 0x400000, v16
@@ -154637,459 +154205,432 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_or_b32_e32 v60, 0x400000, v48
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v48, v48
; GFX9-NEXT: v_bfe_u32 v48, v15, 16, 1
-; GFX9-NEXT: v_perm_b32 v37, v18, v20, s7
-; GFX9-NEXT: v_perm_b32 v36, v8, v17, s7
-; GFX9-NEXT: v_perm_b32 v35, v7, v6, s7
; GFX9-NEXT: v_cndmask_b32_e32 v46, v47, v60, vcc
; GFX9-NEXT: v_add3_u32 v48, v48, v15, s6
; GFX9-NEXT: v_or_b32_e32 v47, 0x400000, v15
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v15, v15
; GFX9-NEXT: v_cndmask_b32_e32 v15, v48, v47, vcc
-; GFX9-NEXT: v_mov_b32_e32 v49, v36
-; GFX9-NEXT: v_mov_b32_e32 v48, v35
-; GFX9-NEXT: v_mov_b32_e32 v35, v37
-; GFX9-NEXT: v_mov_b32_e32 v36, v38
-; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; GFX9-NEXT: v_perm_b32 v41, v16, v63, s7
-; GFX9-NEXT: v_perm_b32 v40, v15, v46, s7
+; GFX9-NEXT: v_perm_b32 v49, v16, v63, s7
+; GFX9-NEXT: v_perm_b32 v48, v15, v46, s7
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v15
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v63
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v46
-; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[40:41]
-; GFX9-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v60, 16, v30
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[48:49]
+; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[31:32]
+; GFX9-NEXT: v_perm_b32 v40, v8, v17, s7
+; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v17
+; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshrrev_b32_e32 v50, 16, v30
; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v42
; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v43
-; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v63
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[31:32]
-; GFX9-NEXT: v_perm_b32 v58, v11, v2, s7
-; GFX9-NEXT: v_perm_b32 v57, v10, v5, s7
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v17
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX9-NEXT: s_waitcnt vmcnt(23)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v20
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v29
-; GFX9-NEXT: v_lshrrev_b32_e32 v29, 16, v39
-; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v62
-; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; GFX9-NEXT: v_perm_b32 v39, v7, v6, s7
+; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v21
+; GFX9-NEXT: v_lshrrev_b32_e32 v51, 16, v51
; GFX9-NEXT: v_lshrrev_b32_e32 v21, 16, v44
-; GFX9-NEXT: v_lshrrev_b32_e32 v44, 8, v48
-; GFX9-NEXT: v_lshrrev_b32_e32 v61, 16, v61
-; GFX9-NEXT: s_waitcnt vmcnt(18)
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v38
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v38
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v37
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: s_waitcnt vmcnt(21)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(10)
+; GFX9-NEXT: v_lshrrev_b32_e32 v63, 8, v43
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshrrev_b32_e32 v60, 24, v14
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[37:38]
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[58:59]
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_lshrrev_b32_e32 v23, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[33:34]
-; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[33:34]
-; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[50:51]
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 8, v33
-; GFX9-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
+; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[33:34]
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[62:63]
+; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[61:62]
+; GFX9-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v62
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[61:62]
+; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v44, 8, v62
+; GFX9-NEXT: s_waitcnt vmcnt(4)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[56:57]
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[48:49]
+; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[56:57]
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v0
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[39:40]
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[35:36]
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[54:55]
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[42:43]
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[50:51]
-; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b64 v[15:16], 24, v[13:14]
+; GFX9-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v55
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v43
+; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshrrev_b32_e32 v54, 8, v54
-; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(18)
+; GFX9-NEXT: s_waitcnt vmcnt(14)
; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v41
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v41
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v40
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v49
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v49
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v48
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v32
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v32
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v31
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v59
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v59
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v58
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v49
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v40
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 24, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v31, 8, v49
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b32_e32 v49, 8, v36
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 8, v35
-; GFX9-NEXT: v_mov_b32_e32 v35, v52
-; GFX9-NEXT: v_mov_b32_e32 v36, v53
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v35
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(15)
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshrrev_b32_e32 v32, 8, v39
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v38
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v38
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 8, v37
+; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v52
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v31, 8, v40
+; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(18)
; GFX9-NEXT: v_lshrrev_b64 v[46:47], 24, v[15:16]
-; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-NEXT: v_lshrrev_b64 v[58:59], 24, v[52:53]
+; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v55
; GFX9-NEXT: v_lshrrev_b32_e32 v59, 24, v57
; GFX9-NEXT: v_lshrrev_b32_e32 v47, 8, v57
; GFX9-NEXT: v_lshrrev_b32_e32 v46, 8, v56
-; GFX9-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; GFX9-NEXT: v_mov_b32_e32 v53, v44
-; GFX9-NEXT: v_lshrrev_b32_e32 v44, 8, v63
-; GFX9-NEXT: v_lshrrev_b32_e32 v32, 8, v55
-; GFX9-NEXT: v_lshrrev_b32_e32 v40, 8, v43
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v56, 8, v55
+; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v43
; GFX9-NEXT: v_lshrrev_b32_e32 v55, 8, v42
-; GFX9-NEXT: s_waitcnt vmcnt(9)
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v53
+; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v53
+; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v33
+; GFX9-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v62
+; GFX9-NEXT: v_lshrrev_b32_e32 v62, 8, v61
+; GFX9-NEXT: v_lshrrev_b32_e32 v61, 8, v14
+; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v13
+; GFX9-NEXT: s_waitcnt vmcnt(12)
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 24, v36
-; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; GFX9-NEXT: v_lshrrev_b32_e32 v36, 8, v36
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v35
; GFX9-NEXT: v_lshrrev_b32_e32 v35, 24, v34
-; GFX9-NEXT: v_lshrrev_b32_e32 v36, 8, v36
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_lshrrev_b32_e32 v35, 8, v39
-; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; GFX9-NEXT: v_mov_b32_e32 v35, v50
-; GFX9-NEXT: v_mov_b32_e32 v36, v51
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v39
-; GFX9-NEXT: v_lshrrev_b32_e32 v52, 8, v38
-; GFX9-NEXT: v_lshrrev_b32_e32 v39, 24, v63
-; GFX9-NEXT: v_lshrrev_b32_e32 v41, 24, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v63, 8, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v38, 8, v35
+; GFX9-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshrrev_b32_e32 v34, 8, v34
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshrrev_b32_e32 v57, 8, v35
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshrrev_b32_e32 v33, 24, v36
+; GFX9-NEXT: v_lshrrev_b32_e32 v42, 8, v36
; GFX9-NEXT: v_mov_b32_e32 v36, v16
; GFX9-NEXT: v_mov_b32_e32 v35, v15
-; GFX9-NEXT: v_lshrrev_b32_e32 v34, 8, v34
; GFX9-NEXT: v_mov_b32_e32 v16, v46
; GFX9-NEXT: v_mov_b32_e32 v46, v37
; GFX9-NEXT: v_lshrrev_b32_e32 v48, 24, v36
; GFX9-NEXT: v_lshrrev_b32_e32 v37, 8, v36
-; GFX9-NEXT: v_lshrrev_b32_e32 v42, 8, v35
+; GFX9-NEXT: v_lshrrev_b32_e32 v53, 8, v35
; GFX9-NEXT: .LBB90_4: ; %end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
; GFX9-NEXT: v_perm_b32 v1, v1, v55, s4
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v54, s4
; GFX9-NEXT: v_perm_b32 v0, v5, v0, s4
; GFX9-NEXT: v_perm_b32 v5, v6, v49, s4
; GFX9-NEXT: v_perm_b32 v8, v8, v31, s4
+; GFX9-NEXT: v_perm_b32 v7, v7, v32, s4
; GFX9-NEXT: v_perm_b32 v10, v10, v47, s4
-; GFX9-NEXT: v_perm_b32 v4, v4, v32, s4
; GFX9-NEXT: v_perm_b32 v9, v9, v16, s4
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v4, v4, v56, s4
; GFX9-NEXT: v_perm_b32 v18, v18, v37, s4
-; GFX9-NEXT: v_perm_b32 v2, v2, v40, s4
-; GFX9-NEXT: v_perm_b32 v7, v7, v53, s4
-; GFX9-NEXT: v_perm_b32 v17, v17, v42, s4
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_perm_b32 v55, v55, v35, s4
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_perm_b32 v32, v32, v15, s4
-; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v54, v54, v35, s4
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-NEXT: v_lshl_or_b32 v0, v54, 16, v0
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v49, v49, v15, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v6, v6, v35, s4
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v2, v2, v63, s4
+; GFX9-NEXT: v_perm_b32 v17, v17, v53, s4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v31, v31, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v43, v43, v15, s4
+; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v11, v11, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v55, v55, v15, s4
+; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v12, v12, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(3)
+; GFX9-NEXT: v_perm_b32 v54, v54, v15, s4
+; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_perm_b32 v13, v13, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v14, v14, v35, s4
-; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; GFX9-NEXT: v_lshl_or_b32 v0, v54, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v6, v6, v15, s4
+; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v14, v14, v35, s4
+; GFX9-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v31, v31, v15, s4
+; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v47, v47, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v32, v32, v15, s4
+; GFX9-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v49, v49, v15, s4
+; GFX9-NEXT: v_perm_b32 v15, v40, v35, s4
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v15, v56, v35, s4
+; GFX9-NEXT: v_perm_b32 v16, v41, v35, s4
; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v43, v43, v50, s4
-; GFX9-NEXT: v_lshl_or_b32 v1, v43, 16, v1
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v16, v57, v35, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v56, v35, v50, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v56, v35, v36, s4
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v37, v35, v36, s4
; GFX9-NEXT: buffer_store_dword v1, v45, s[0:3], 0 offen
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v35, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
; GFX9-NEXT: buffer_store_dword v1, v45, s[0:3], 0 offen offset:4
; GFX9-NEXT: v_lshl_or_b32 v1, v55, 16, v3
; GFX9-NEXT: buffer_store_dword v1, v45, s[0:3], 0 offen offset:8
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
; GFX9-NEXT: buffer_store_dword v1, v45, s[0:3], 0 offen offset:12
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:16
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v5
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:20
; GFX9-NEXT: v_lshl_or_b32 v0, v6, 16, v7
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:24
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v8
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:28
; GFX9-NEXT: v_lshl_or_b32 v0, v31, 16, v9
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:32
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v59, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v10
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:36
; GFX9-NEXT: v_lshl_or_b32 v0, v32, 16, v11
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:40
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v12
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:44
; GFX9-NEXT: v_lshl_or_b32 v0, v49, 16, v13
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:48
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v14
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:52
; GFX9-NEXT: v_lshl_or_b32 v0, v47, 16, v15
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:56
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v16
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:60
; GFX9-NEXT: v_lshl_or_b32 v0, v56, 16, v17
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:64
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v0, v0, v48, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v18
@@ -155097,113 +154638,104 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v0, v19, v38, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v37, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:72
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v0, v20, v63, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v20, v61, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v1, v41, s4
+; GFX9-NEXT: v_perm_b32 v1, v1, v60, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:76
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, v21, v62, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:80
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, v22, v44, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v39, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:84
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v0, v23, v52, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v23, v57, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:88
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v24, v0, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v24, v42, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v33, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:92
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v0, v25, v52, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_perm_b32 v0, v25, v0, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:96
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, v26, v34, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:100
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v0, v27, v46, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:104
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v0, v28, v0, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:108
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_perm_b32 v0, v29, v0, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v58, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:112
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
; GFX9-NEXT: v_perm_b32 v0, v30, v0, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:116
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v0, v50, v0, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
-; GFX9-NEXT: v_perm_b32 v0, v60, v0, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: buffer_store_dword v0, v45, s[0:3], 0 offen offset:120
-; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v0, v61, v0, s4
+; GFX9-NEXT: v_perm_b32 v0, v51, v0, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -157301,9 +156833,9 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
@@ -157354,36 +156886,36 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v63, s99, 33
; SI-NEXT: v_writelane_b32 v63, s30, 34
; SI-NEXT: v_writelane_b32 v63, s31, 35
-; SI-NEXT: v_readfirstlane_b32 s58, v14
; SI-NEXT: v_readfirstlane_b32 s56, v15
-; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: v_readfirstlane_b32 s88, v7
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s59
; SI-NEXT: v_readfirstlane_b32 s46, v16
; SI-NEXT: s_lshl_b32 s56, s56, 16
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s57
-; SI-NEXT: v_readfirstlane_b32 s42, v18
+; SI-NEXT: v_readfirstlane_b32 s90, v6
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s57
; SI-NEXT: v_readfirstlane_b32 s44, v17
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s56
+; SI-NEXT: v_readfirstlane_b32 s42, v18
+; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
+; SI-NEXT: v_readfirstlane_b32 s58, v14
; SI-NEXT: v_readfirstlane_b32 s60, v13
; SI-NEXT: v_readfirstlane_b32 s62, v12
; SI-NEXT: v_readfirstlane_b32 s72, v11
; SI-NEXT: v_readfirstlane_b32 s74, v10
; SI-NEXT: v_readfirstlane_b32 s76, v9
; SI-NEXT: v_readfirstlane_b32 s78, v8
-; SI-NEXT: v_readfirstlane_b32 s90, v6
+; SI-NEXT: v_readfirstlane_b32 s88, v7
; SI-NEXT: v_readfirstlane_b32 s92, v5
; SI-NEXT: v_readfirstlane_b32 s94, v4
; SI-NEXT: v_readfirstlane_b32 vcc_lo, v3
; SI-NEXT: v_readfirstlane_b32 s30, v2
; SI-NEXT: v_readfirstlane_b32 s34, v1
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s56
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s46
; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
; SI-NEXT: s_lshl_b32 s5, s29, 16
; SI-NEXT: s_and_b32 s6, s28, 0xffff0000
@@ -157414,8 +156946,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
; SI-NEXT: s_lshl_b32 s42, s42, 16
-; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
; SI-NEXT: s_lshl_b32 s44, s44, 16
+; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
; SI-NEXT: s_lshl_b32 s58, s58, 16
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
; SI-NEXT: s_lshl_b32 s60, s60, 16
@@ -157444,236 +156976,246 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
; SI-NEXT: s_lshl_b32 s34, s34, 16
; SI-NEXT: v_readfirstlane_b32 s36, v19
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s46
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s45
; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v10, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v14, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s40
-; SI-NEXT: v_mul_f32_e64 v3, 1.0, s17
-; SI-NEXT: v_mul_f32_e64 v21, 1.0, s29
-; SI-NEXT: v_mul_f32_e64 v25, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v8, 1.0, s28
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v15, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v4, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v7, 1.0, s17
+; SI-NEXT: v_mul_f32_e64 v23, 1.0, s29
+; SI-NEXT: v_mul_f32_e64 v26, 1.0, s18
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v18, 1.0, s19
-; SI-NEXT: v_mul_f32_e64 v27, 1.0, s27
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v54, 1.0, s26
-; SI-NEXT: v_mul_f32_e64 v26, 1.0, s21
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, s25
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v15, 1.0, s24
+; SI-NEXT: v_mul_f32_e64 v28, 1.0, s27
+; SI-NEXT: v_mul_f32_e64 v30, 1.0, s20
+; SI-NEXT: v_mul_f32_e64 v9, 1.0, s26
+; SI-NEXT: v_mul_f32_e64 v27, 1.0, s21
+; SI-NEXT: v_mul_f32_e64 v36, 1.0, s25
+; SI-NEXT: v_mul_f32_e64 v49, 1.0, s22
+; SI-NEXT: v_mul_f32_e64 v14, 1.0, s24
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s23
-; SI-NEXT: v_mul_f32_e64 v59, 1.0, s14
-; SI-NEXT: v_mul_f32_e64 v29, 1.0, s15
+; SI-NEXT: v_mul_f32_e64 v55, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v57, 1.0, s15
; SI-NEXT: v_mul_f32_e64 v20, 1.0, s12
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s13
-; SI-NEXT: v_mul_f32_e64 v31, 1.0, s10
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s11
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s8
-; SI-NEXT: v_mul_f32_e64 v30, 1.0, s9
-; SI-NEXT: v_mul_f32_e64 v53, 1.0, s6
-; SI-NEXT: v_mul_f32_e64 v55, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s4
+; SI-NEXT: v_mul_f32_e64 v50, 1.0, s13
+; SI-NEXT: v_mul_f32_e64 v29, 1.0, s10
+; SI-NEXT: v_mul_f32_e64 v48, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v59, 1.0, s8
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s9
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s6
+; SI-NEXT: v_mul_f32_e64 v54, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v22, 1.0, s4
; SI-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; SI-NEXT: v_mul_f32_e64 v36, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v33, 1.0, s35
; SI-NEXT: v_mul_f32_e64 v41, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v13, 1.0, s30
-; SI-NEXT: v_mul_f32_e64 v49, 1.0, vcc_hi
-; SI-NEXT: v_mul_f32_e64 v12, 1.0, vcc_lo
-; SI-NEXT: v_mul_f32_e64 v24, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v28, 1.0, s94
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, s31
+; SI-NEXT: v_mul_f32_e64 v32, 1.0, s30
+; SI-NEXT: v_mul_f32_e64 v39, 1.0, vcc_hi
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, vcc_lo
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, s95
+; SI-NEXT: v_mul_f32_e64 v17, 1.0, s94
; SI-NEXT: v_mul_f32_e64 v43, 1.0, s93
-; SI-NEXT: v_mul_f32_e64 v46, 1.0, s92
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s91
-; SI-NEXT: v_mul_f32_e64 v42, 1.0, s90
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s89
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s92
+; SI-NEXT: v_mul_f32_e64 v40, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v31, 1.0, s90
+; SI-NEXT: v_mul_f32_e64 v47, 1.0, s89
; SI-NEXT: v_mul_f32_e64 v58, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s79
-; SI-NEXT: v_mul_f32_e64 v47, 1.0, s78
-; SI-NEXT: v_mul_f32_e64 v33, 1.0, s77
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s79
+; SI-NEXT: v_mul_f32_e64 v13, 1.0, s78
+; SI-NEXT: v_mul_f32_e64 v16, 1.0, s77
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s75
-; SI-NEXT: v_mul_f32_e64 v57, 1.0, s74
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s75
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s74
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v22, 1.0, s72
-; SI-NEXT: v_mul_f32_e64 v11, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v4, 1.0, s62
-; SI-NEXT: v_mul_f32_e64 v16, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v17, 1.0, s60
-; SI-NEXT: v_mul_f32_e64 v9, 1.0, s58
+; SI-NEXT: v_mul_f32_e64 v11, 1.0, s72
+; SI-NEXT: v_mul_f32_e64 v3, 1.0, s63
+; SI-NEXT: v_mul_f32_e64 v21, 1.0, s62
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s61
+; SI-NEXT: v_mul_f32_e64 v25, 1.0, s60
+; SI-NEXT: v_mul_f32_e64 v10, 1.0, s59
+; SI-NEXT: v_mul_f32_e64 v42, 1.0, s58
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s47
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s45
-; SI-NEXT: v_mul_f32_e64 v48, 1.0, s44
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v34, 1.0, s44
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v7, 1.0, s43
-; SI-NEXT: v_mul_f32_e64 v34, 1.0, s42
+; SI-NEXT: v_mul_f32_e64 v6, 1.0, s43
+; SI-NEXT: v_mul_f32_e64 v53, 1.0, s42
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr62 : SGPR spill to VGPR lane
; SI-NEXT: ; implicit-def: $vgpr61 : SGPR spill to VGPR lane
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB91_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v14
+; SI-NEXT: v_readfirstlane_b32 s4, v15
; SI-NEXT: s_lshr_b64 s[98:99], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v6
+; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: s_lshr_b32 s87, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v21
+; SI-NEXT: v_readfirstlane_b32 s4, v23
+; SI-NEXT: v_readfirstlane_b32 s86, v7
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v25
+; SI-NEXT: v_readfirstlane_b32 s4, v26
+; SI-NEXT: v_mov_b32_e32 v7, v46
; SI-NEXT: s_lshr_b64 s[84:85], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v8
+; SI-NEXT: v_readfirstlane_b32 s4, v7
; SI-NEXT: s_lshr_b32 s51, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v27
-; SI-NEXT: v_mov_b32_e32 v27, v55
-; SI-NEXT: v_mov_b32_e32 v55, v36
-; SI-NEXT: v_mov_b32_e32 v36, v47
-; SI-NEXT: v_mov_b32_e32 v47, v57
-; SI-NEXT: v_mov_b32_e32 v57, v11
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s74, v35
-; SI-NEXT: v_readfirstlane_b32 s88, v30
-; SI-NEXT: v_mov_b32_e32 v30, v28
-; SI-NEXT: v_mov_b32_e32 v35, v42
-; SI-NEXT: v_mov_b32_e32 v42, v40
-; SI-NEXT: v_mov_b32_e32 v28, v22
-; SI-NEXT: v_mov_b32_e32 v22, v4
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s4, v28
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v32
-; SI-NEXT: v_mov_b32_e32 v32, v2
-; SI-NEXT: v_mov_b32_e32 v2, v12
-; SI-NEXT: v_mov_b32_e32 v12, v33
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: v_readfirstlane_b32 s50, v18
-; SI-NEXT: v_mov_b32_e32 v18, v54
+; SI-NEXT: v_readfirstlane_b32 s4, v30
; SI-NEXT: s_lshr_b64 s[70:71], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v18
+; SI-NEXT: v_readfirstlane_b32 s4, v9
; SI-NEXT: s_lshr_b32 s67, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v38
+; SI-NEXT: v_readfirstlane_b32 s4, v36
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v50
+; SI-NEXT: v_readfirstlane_b32 s4, v49
; SI-NEXT: s_lshr_b64 s[64:65], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v15
+; SI-NEXT: v_readfirstlane_b32 s4, v14
; SI-NEXT: s_lshr_b32 s75, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v59
+; SI-NEXT: v_readfirstlane_b32 s4, v55
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v29
+; SI-NEXT: v_readfirstlane_b32 s4, v57
; SI-NEXT: s_lshr_b64 s[48:49], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v20
; SI-NEXT: s_lshr_b32 s43, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v31
+; SI-NEXT: v_readfirstlane_b32 s4, v29
+; SI-NEXT: v_mov_b32_e32 v29, v16
+; SI-NEXT: v_mov_b32_e32 v16, v8
+; SI-NEXT: v_mov_b32_e32 v8, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_readfirstlane_b32 s74, v35
+; SI-NEXT: v_mov_b32_e32 v30, v54
+; SI-NEXT: v_mov_b32_e32 v54, v39
+; SI-NEXT: v_mov_b32_e32 v35, v17
+; SI-NEXT: v_mov_b32_e32 v39, v43
+; SI-NEXT: v_mov_b32_e32 v43, v31
+; SI-NEXT: v_mov_b32_e32 v17, v58
+; SI-NEXT: v_mov_b32_e32 v58, v13
+; SI-NEXT: v_mov_b32_e32 v13, v24
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v28, v52
+; SI-NEXT: v_mov_b32_e32 v52, v41
+; SI-NEXT: v_mov_b32_e32 v41, v38
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v36, v2
+; SI-NEXT: v_mov_b32_e32 v2, v32
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v49, v48
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v51
+; SI-NEXT: v_readfirstlane_b32 s4, v49
; SI-NEXT: s_lshr_b64 s[94:95], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v60
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v51, v53
+; SI-NEXT: v_readfirstlane_b32 s4, v59
; SI-NEXT: s_lshr_b32 s89, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v51
+; SI-NEXT: v_readfirstlane_b32 s4, v28
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v27
+; SI-NEXT: v_readfirstlane_b32 s4, v30
; SI-NEXT: s_lshr_b64 s[76:77], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v23
+; SI-NEXT: v_readfirstlane_b32 s4, v22
+; SI-NEXT: v_mov_b32_e32 v48, v33
; SI-NEXT: s_lshr_b32 s37, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v55
-; SI-NEXT: v_mov_b32_e32 v53, v41
+; SI-NEXT: v_readfirstlane_b32 s4, v48
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v53
+; SI-NEXT: v_readfirstlane_b32 s4, v52
; SI-NEXT: s_lshr_b64 s[30:31], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v39
-; SI-NEXT: v_mov_b32_e32 v41, v49
+; SI-NEXT: v_readfirstlane_b32 s4, v37
; SI-NEXT: s_lshr_b32 s15, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v41
+; SI-NEXT: v_readfirstlane_b32 s4, v54
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v2
+; SI-NEXT: v_readfirstlane_b32 s4, v41
; SI-NEXT: s_lshr_b64 s[78:79], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v24
+; SI-NEXT: v_readfirstlane_b32 s4, v51
; SI-NEXT: s_lshr_b32 s73, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v43
+; SI-NEXT: v_readfirstlane_b32 s4, v39
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v46
+; SI-NEXT: v_readfirstlane_b32 s4, v45
; SI-NEXT: s_lshr_b64 s[62:63], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v52
+; SI-NEXT: v_readfirstlane_b32 s4, v40
; SI-NEXT: s_lshr_b32 s59, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v56
+; SI-NEXT: v_readfirstlane_b32 s4, v47
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v58
+; SI-NEXT: v_readfirstlane_b32 s4, v17
; SI-NEXT: s_lshr_b64 s[46:47], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v42
+; SI-NEXT: v_readfirstlane_b32 s4, v44
; SI-NEXT: s_lshr_b32 s45, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v12
+; SI-NEXT: v_readfirstlane_b32 s4, v29
; SI-NEXT: s_lshr_b32 s5, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v1
; SI-NEXT: s_lshr_b64 s[40:41], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v44
+; SI-NEXT: v_readfirstlane_b32 s4, v56
; SI-NEXT: s_lshr_b32 s29, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v5
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v28
+; SI-NEXT: v_readfirstlane_b32 s4, v11
; SI-NEXT: s_lshr_b64 s[24:25], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s4, v57
+; SI-NEXT: v_readfirstlane_b32 s4, v3
; SI-NEXT: s_lshr_b32 s23, s4, 16
; SI-NEXT: v_readfirstlane_b32 s4, v16
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v17
+; SI-NEXT: v_readfirstlane_b32 s4, v8
; SI-NEXT: s_lshr_b64 s[16:17], s[4:5], 16
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_readfirstlane_b32 s4, v11
+; SI-NEXT: v_readfirstlane_b32 s4, v10
; SI-NEXT: s_lshr_b32 s19, s4, 16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s14, v2
+; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
+; SI-NEXT: s_lshr_b64 s[96:97], s[86:87], 16
+; SI-NEXT: s_mov_b32 s99, s96
+; SI-NEXT: v_readfirstlane_b32 s50, v18
+; SI-NEXT: s_lshr_b64 s[82:83], s[50:51], 16
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_readfirstlane_b32 s4, v25
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_mov_b32_e32 v29, v13
-; SI-NEXT: v_readfirstlane_b32 s14, v29
-; SI-NEXT: v_readfirstlane_b32 s86, v3
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_readfirstlane_b32 s4, v33
+; SI-NEXT: s_mov_b32 s85, s82
+; SI-NEXT: v_readfirstlane_b32 s66, v27
+; SI-NEXT: s_lshr_b64 s[68:69], s[66:67], 16
+; SI-NEXT: s_mov_b32 s71, s68
+; SI-NEXT: s_lshr_b64 s[54:55], s[74:75], 16
+; SI-NEXT: s_mov_b32 s65, s54
+; SI-NEXT: v_readfirstlane_b32 s42, v50
+; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
+; SI-NEXT: s_mov_b32 s49, vcc_lo
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_readfirstlane_b32 s4, v31
; SI-NEXT: s_lshr_b64 s[10:11], s[4:5], 16
; SI-NEXT: v_readfirstlane_b32 s4, v19
; SI-NEXT: s_lshr_b32 s13, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v37
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_readfirstlane_b32 s4, v38
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v48
+; SI-NEXT: v_readfirstlane_b32 s4, v34
; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_readfirstlane_b32 s5, v7
-; SI-NEXT: s_lshr_b64 s[92:93], s[14:15], 16
+; SI-NEXT: v_readfirstlane_b32 s5, v6
; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v34
-; SI-NEXT: s_lshr_b64 s[96:97], s[86:87], 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_readfirstlane_b32 s6, v32
; SI-NEXT: s_mov_b32 s5, s7
; SI-NEXT: s_lshr_b64 s[8:9], s[6:7], 16
; SI-NEXT: s_mov_b32 s6, s92
-; SI-NEXT: s_mov_b32 s99, s96
; SI-NEXT: v_writelane_b32 v61, s4, 34
; SI-NEXT: v_writelane_b32 v62, s6, 48
; SI-NEXT: v_writelane_b32 v61, s5, 35
@@ -157686,22 +157228,17 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s9, 5
; SI-NEXT: s_lshr_b64 s[8:9], s[98:99], 16
; SI-NEXT: v_writelane_b32 v62, s8, 2
-; SI-NEXT: s_lshr_b64 s[82:83], s[50:51], 16
; SI-NEXT: v_writelane_b32 v62, s9, 3
; SI-NEXT: s_lshr_b64 s[8:9], s[98:99], 8
-; SI-NEXT: s_mov_b32 s85, s82
; SI-NEXT: v_writelane_b32 v62, s8, 0
; SI-NEXT: v_writelane_b32 v62, s9, 1
; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 24
; SI-NEXT: v_writelane_b32 v62, s8, 10
; SI-NEXT: v_writelane_b32 v62, s9, 11
; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 16
-; SI-NEXT: v_readfirstlane_b32 s66, v26
; SI-NEXT: v_writelane_b32 v62, s8, 8
-; SI-NEXT: s_lshr_b64 s[68:69], s[66:67], 16
; SI-NEXT: v_writelane_b32 v62, s9, 9
; SI-NEXT: s_lshr_b64 s[8:9], s[84:85], 8
-; SI-NEXT: s_mov_b32 s71, s68
; SI-NEXT: v_writelane_b32 v62, s8, 6
; SI-NEXT: v_writelane_b32 v62, s9, 7
; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 24
@@ -157709,28 +157246,24 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s9, 17
; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 16
; SI-NEXT: v_writelane_b32 v62, s8, 14
-; SI-NEXT: s_lshr_b64 s[54:55], s[74:75], 16
; SI-NEXT: v_writelane_b32 v62, s9, 15
; SI-NEXT: s_lshr_b64 s[8:9], s[70:71], 8
-; SI-NEXT: s_mov_b32 s65, s54
; SI-NEXT: v_writelane_b32 v62, s8, 12
; SI-NEXT: v_writelane_b32 v62, s9, 13
; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 24
; SI-NEXT: v_writelane_b32 v62, s8, 22
; SI-NEXT: v_writelane_b32 v62, s9, 23
; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 16
-; SI-NEXT: v_readfirstlane_b32 s42, v45
; SI-NEXT: v_writelane_b32 v62, s8, 20
-; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
; SI-NEXT: v_writelane_b32 v62, s9, 21
; SI-NEXT: s_lshr_b64 s[8:9], s[64:65], 8
-; SI-NEXT: s_mov_b32 s49, vcc_lo
; SI-NEXT: v_writelane_b32 v62, s8, 18
; SI-NEXT: v_writelane_b32 v62, s9, 19
; SI-NEXT: s_lshr_b64 s[8:9], s[48:49], 24
; SI-NEXT: v_writelane_b32 v62, s8, 28
; SI-NEXT: v_writelane_b32 v62, s9, 29
; SI-NEXT: s_lshr_b64 s[8:9], s[48:49], 16
+; SI-NEXT: v_readfirstlane_b32 s88, v60
; SI-NEXT: v_writelane_b32 v62, s8, 26
; SI-NEXT: s_lshr_b64 s[80:81], s[88:89], 16
; SI-NEXT: v_writelane_b32 v62, s9, 27
@@ -157742,7 +157275,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s8, 34
; SI-NEXT: v_writelane_b32 v62, s9, 35
; SI-NEXT: s_lshr_b64 s[8:9], s[94:95], 16
-; SI-NEXT: v_readfirstlane_b32 s36, v32
+; SI-NEXT: v_readfirstlane_b32 s36, v36
; SI-NEXT: v_writelane_b32 v62, s8, 32
; SI-NEXT: s_lshr_b64 s[38:39], s[36:37], 16
; SI-NEXT: v_writelane_b32 v62, s9, 33
@@ -157764,7 +157297,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s8, 46
; SI-NEXT: v_writelane_b32 v62, s9, 47
; SI-NEXT: s_lshr_b64 s[8:9], s[30:31], 16
-; SI-NEXT: v_readfirstlane_b32 s72, v30
+; SI-NEXT: v_readfirstlane_b32 s72, v35
; SI-NEXT: v_writelane_b32 v62, s8, 44
; SI-NEXT: s_lshr_b64 s[60:61], s[72:73], 16
; SI-NEXT: v_writelane_b32 v62, s9, 45
@@ -157776,7 +157309,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s8, 54
; SI-NEXT: v_writelane_b32 v62, s9, 55
; SI-NEXT: s_lshr_b64 s[8:9], s[78:79], 16
-; SI-NEXT: v_readfirstlane_b32 s58, v35
+; SI-NEXT: v_readfirstlane_b32 s58, v43
; SI-NEXT: v_writelane_b32 v62, s8, 52
; SI-NEXT: s_lshr_b64 s[56:57], s[58:59], 16
; SI-NEXT: v_writelane_b32 v62, s9, 53
@@ -157788,7 +157321,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s8, 60
; SI-NEXT: v_writelane_b32 v62, s9, 61
; SI-NEXT: s_lshr_b64 s[8:9], s[62:63], 16
-; SI-NEXT: v_readfirstlane_b32 s44, v36
+; SI-NEXT: v_readfirstlane_b32 s44, v58
; SI-NEXT: v_writelane_b32 v62, s8, 58
; SI-NEXT: s_lshr_b64 s[52:53], s[44:45], 16
; SI-NEXT: v_writelane_b32 v62, s9, 59
@@ -157800,7 +157333,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s8, 2
; SI-NEXT: v_writelane_b32 v61, s9, 3
; SI-NEXT: s_lshr_b64 s[8:9], s[46:47], 16
-; SI-NEXT: v_readfirstlane_b32 s28, v47
+; SI-NEXT: v_readfirstlane_b32 s28, v13
; SI-NEXT: v_writelane_b32 v61, s8, 0
; SI-NEXT: s_lshr_b64 s[26:27], s[28:29], 16
; SI-NEXT: v_writelane_b32 v61, s9, 1
@@ -157810,9 +157343,11 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v62, s9, 63
; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 24
; SI-NEXT: v_writelane_b32 v61, s8, 8
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_mov_b32_e32 v5, v21
; SI-NEXT: v_writelane_b32 v61, s9, 9
; SI-NEXT: s_lshr_b64 s[8:9], s[40:41], 16
-; SI-NEXT: v_readfirstlane_b32 s22, v22
+; SI-NEXT: v_readfirstlane_b32 s22, v5
; SI-NEXT: v_writelane_b32 v61, s8, 6
; SI-NEXT: s_lshr_b64 s[20:21], s[22:23], 16
; SI-NEXT: v_writelane_b32 v61, s9, 7
@@ -157822,9 +157357,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s9, 5
; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 24
; SI-NEXT: v_writelane_b32 v61, s8, 14
+; SI-NEXT: v_mov_b32_e32 v21, v42
; SI-NEXT: v_writelane_b32 v61, s9, 15
; SI-NEXT: s_lshr_b64 s[8:9], s[24:25], 16
-; SI-NEXT: v_readfirstlane_b32 s18, v9
+; SI-NEXT: v_readfirstlane_b32 s18, v21
; SI-NEXT: v_writelane_b32 v61, s8, 12
; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
; SI-NEXT: v_writelane_b32 v61, s9, 13
@@ -157836,7 +157372,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s8, 20
; SI-NEXT: v_writelane_b32 v61, s9, 21
; SI-NEXT: s_lshr_b64 s[8:9], s[16:17], 16
-; SI-NEXT: v_readfirstlane_b32 s12, v40
+; SI-NEXT: v_readfirstlane_b32 s12, v24
; SI-NEXT: v_writelane_b32 v61, s8, 18
; SI-NEXT: s_lshr_b64 s[90:91], s[12:13], 16
; SI-NEXT: v_writelane_b32 v61, s9, 19
@@ -157858,20 +157394,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s9, 33
; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 16
; SI-NEXT: v_writelane_b32 v61, s8, 30
-; SI-NEXT: v_mov_b32_e32 v1, v8
-; SI-NEXT: v_mov_b32_e32 v8, v5
; SI-NEXT: v_writelane_b32 v61, s9, 31
; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], 8
-; SI-NEXT: v_lshrrev_b32_e32 v5, 24, v6
-; SI-NEXT: v_mov_b32_e32 v3, v20
+; SI-NEXT: v_mov_b32_e32 v1, v9
; SI-NEXT: v_writelane_b32 v61, s8, 28
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v5, 24, v57
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v11
-; SI-NEXT: v_lshrrev_b32_e32 v6, 24, v7
-; SI-NEXT: v_mov_b32_e32 v20, v17
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_mov_b32_e32 v9, v11
; SI-NEXT: s_lshr_b32 s88, s96, 8
; SI-NEXT: s_lshr_b32 s61, s82, 8
; SI-NEXT: s_lshr_b32 s72, s68, 8
@@ -157894,147 +157422,161 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b32 s12, s90, 8
; SI-NEXT: v_writelane_b32 v61, s9, 29
; SI-NEXT: s_mov_b64 s[8:9], 0
-; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v18
-; SI-NEXT: v_mov_b32_e32 v16, v15
-; SI-NEXT: v_lshrrev_b32_e32 v21, 24, v15
-; SI-NEXT: v_lshrrev_b32_e32 v17, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v14, 24, v60
-; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v23
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v39
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v24
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v52
-; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v42
-; SI-NEXT: v_lshrrev_b32_e32 v38, 24, v44
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v19
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v48, v32
-; SI-NEXT: v_mov_b32_e32 v32, v5
-; SI-NEXT: v_mov_b32_e32 v34, v51
-; SI-NEXT: v_mov_b32_e32 v37, v27
-; SI-NEXT: v_mov_b32_e32 v27, v1
-; SI-NEXT: v_mov_b32_e32 v51, v55
-; SI-NEXT: v_mov_b32_e32 v55, v41
-; SI-NEXT: v_mov_b32_e32 v41, v2
-; SI-NEXT: v_mov_b32_e32 v31, v12
-; SI-NEXT: v_mov_b32_e32 v1, v22
-; SI-NEXT: v_mov_b32_e32 v12, v28
-; SI-NEXT: v_mov_b32_e32 v5, v54
-; SI-NEXT: v_mov_b32_e32 v22, v40
-; SI-NEXT: v_mov_b32_e32 v28, v4
-; SI-NEXT: v_mov_b32_e32 v40, v42
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v6, v57
-; SI-NEXT: v_mov_b32_e32 v4, v11
+; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v1
+; SI-NEXT: v_lshrrev_b32_e32 v18, 24, v14
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v20
+; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v59
+; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v22
+; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v37
+; SI-NEXT: v_lshrrev_b32_e32 v57, 24, v51
+; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v40
+; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v44
+; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v56
+; SI-NEXT: v_lshrrev_b32_e32 v46, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v53, 24, v19
+; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v6
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v20, v49
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v34, v36
+; SI-NEXT: v_mov_b32_e32 v14, v2
+; SI-NEXT: v_mov_b32_e32 v36, v43
+; SI-NEXT: v_mov_b32_e32 v43, v39
+; SI-NEXT: v_mov_b32_e32 v39, v58
+; SI-NEXT: v_mov_b32_e32 v58, v17
+; SI-NEXT: v_mov_b32_e32 v49, v13
+; SI-NEXT: v_mov_b32_e32 v1, v5
+; SI-NEXT: v_mov_b32_e32 v5, v7
+; SI-NEXT: v_mov_b32_e32 v13, v21
+; SI-NEXT: v_mov_b32_e32 v17, v8
+; SI-NEXT: v_mov_b32_e32 v21, v24
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v2, v10
+; SI-NEXT: v_mov_b32_e32 v3, v19
; SI-NEXT: s_branch .LBB91_3
; SI-NEXT: .LBB91_2:
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_mov_b32_e32 v34, v53
+; SI-NEXT: v_mov_b32_e32 v35, v17
; SI-NEXT: v_writelane_b32 v62, s4, 0
; SI-NEXT: v_writelane_b32 v62, s5, 1
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: v_mov_b32_e32 v53, v41
+; SI-NEXT: v_mov_b32_e32 v36, v31
; SI-NEXT: v_writelane_b32 v62, s4, 2
; SI-NEXT: v_writelane_b32 v62, s5, 3
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v30, v28
+; SI-NEXT: s_waitcnt expcnt(6)
+; SI-NEXT: v_mov_b32_e32 v1, v21
; SI-NEXT: v_writelane_b32 v62, s4, 4
; SI-NEXT: v_writelane_b32 v62, s5, 5
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v41, v12
+; SI-NEXT: v_mov_b32_e32 v17, v25
; SI-NEXT: v_writelane_b32 v62, s4, 6
; SI-NEXT: v_writelane_b32 v62, s5, 7
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v31, v33
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 8
; SI-NEXT: v_writelane_b32 v62, s5, 9
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v12, v22
+; SI-NEXT: v_mov_b32_e32 v28, v52
; SI-NEXT: v_writelane_b32 v62, s4, 10
; SI-NEXT: v_writelane_b32 v62, s5, 11
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_mov_b32_e32 v14, v32
; SI-NEXT: v_writelane_b32 v62, s4, 12
; SI-NEXT: v_writelane_b32 v62, s5, 13
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; kill: killed $vgpr1
-; SI-NEXT: v_mov_b32_e32 v1, v4
+; SI-NEXT: v_mov_b32_e32 v52, v41
; SI-NEXT: v_writelane_b32 v62, s4, 14
; SI-NEXT: v_writelane_b32 v62, s5, 15
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v41, v38
; SI-NEXT: v_writelane_b32 v62, s4, 16
; SI-NEXT: v_writelane_b32 v62, s5, 17
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: ; implicit-def: $sgpr15
-; SI-NEXT: ; implicit-def: $sgpr14
-; SI-NEXT: v_mov_b32_e32 v20, v17
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: v_writelane_b32 v62, s4, 18
; SI-NEXT: v_writelane_b32 v62, s5, 19
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v16, v15
+; SI-NEXT: ; implicit-def: $sgpr15
+; SI-NEXT: ; implicit-def: $sgpr14
+; SI-NEXT: v_mov_b32_e32 v9, v11
; SI-NEXT: v_writelane_b32 v62, s4, 20
; SI-NEXT: v_writelane_b32 v62, s5, 21
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v8, v5
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: v_writelane_b32 v62, s4, 22
; SI-NEXT: v_writelane_b32 v62, s5, 23
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_mov_b64 s[8:9], -1
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v20, v48
; SI-NEXT: v_writelane_b32 v62, s4, 24
; SI-NEXT: v_writelane_b32 v62, s5, 25
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v48, v2
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v34, v2
; SI-NEXT: v_writelane_b32 v62, s4, 26
; SI-NEXT: v_writelane_b32 v62, s5, 27
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v37, v55
+; SI-NEXT: v_mov_b32_e32 v30, v54
; SI-NEXT: v_writelane_b32 v62, s4, 28
; SI-NEXT: v_writelane_b32 v62, s5, 29
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v29, v13
+; SI-NEXT: v_mov_b32_e32 v48, v33
; SI-NEXT: v_writelane_b32 v62, s4, 30
; SI-NEXT: v_writelane_b32 v62, s5, 31
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v51, v36
+; SI-NEXT: v_mov_b32_e32 v54, v39
; SI-NEXT: v_writelane_b32 v62, s4, 32
; SI-NEXT: v_writelane_b32 v62, s5, 33
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v55, v49
+; SI-NEXT: v_mov_b32_e32 v39, v13
; SI-NEXT: v_writelane_b32 v62, s4, 34
; SI-NEXT: v_writelane_b32 v62, s5, 35
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v35, v42
+; SI-NEXT: v_mov_b32_e32 v49, v24
; SI-NEXT: v_writelane_b32 v62, s4, 36
; SI-NEXT: v_writelane_b32 v62, s5, 37
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v36, v47
+; SI-NEXT: v_mov_b32_e32 v29, v16
; SI-NEXT: v_writelane_b32 v62, s4, 38
; SI-NEXT: v_writelane_b32 v62, s5, 39
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v47, v57
+; SI-NEXT: v_mov_b32_e32 v5, v46
; SI-NEXT: v_writelane_b32 v62, s4, 40
; SI-NEXT: v_writelane_b32 v62, s5, 41
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v5, v54
+; SI-NEXT: v_mov_b32_e32 v13, v42
; SI-NEXT: v_writelane_b32 v62, s4, 42
; SI-NEXT: v_writelane_b32 v62, s5, 43
; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_mov_b32_e32 v6, v11
+; SI-NEXT: v_mov_b32_e32 v16, v8
; SI-NEXT: v_writelane_b32 v62, s4, 44
; SI-NEXT: v_writelane_b32 v62, s5, 45
; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_writelane_b32 v62, s4, 46
+; SI-NEXT: v_writelane_b32 v62, s5, 47
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_mov_b32_e32 v2, v10
+; SI-NEXT: v_writelane_b32 v62, s4, 48
+; SI-NEXT: v_writelane_b32 v62, s5, 49
+; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: v_mov_b32_e32 v3, v19
+; SI-NEXT: v_writelane_b32 v62, s4, 50
+; SI-NEXT: v_writelane_b32 v62, s5, 51
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: ; implicit-def: $sgpr98
; SI-NEXT: ; implicit-def: $sgpr96
; SI-NEXT: ; implicit-def: $sgpr88
; SI-NEXT: ; implicit-def: $sgpr87
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $sgpr84
; SI-NEXT: ; implicit-def: $sgpr82
; SI-NEXT: ; implicit-def: $sgpr61
@@ -158044,39 +157586,37 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $sgpr68
; SI-NEXT: ; implicit-def: $sgpr72
; SI-NEXT: ; implicit-def: $sgpr67
-; SI-NEXT: ; implicit-def: $vgpr25
+; SI-NEXT: ; implicit-def: $vgpr23
; SI-NEXT: ; implicit-def: $sgpr64
; SI-NEXT: ; implicit-def: $sgpr54
; SI-NEXT: ; implicit-def: $sgpr57
; SI-NEXT: ; implicit-def: $sgpr75
-; SI-NEXT: ; implicit-def: $vgpr21
+; SI-NEXT: ; implicit-def: $vgpr18
; SI-NEXT: ; implicit-def: $sgpr48
; SI-NEXT: ; implicit-def: $vcc_lo
; SI-NEXT: ; implicit-def: $sgpr58
; SI-NEXT: ; implicit-def: $sgpr43
-; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr15
; SI-NEXT: ; implicit-def: $sgpr94
; SI-NEXT: ; implicit-def: $sgpr80
; SI-NEXT: ; implicit-def: $sgpr93
; SI-NEXT: ; implicit-def: $sgpr89
-; SI-NEXT: ; implicit-def: $vgpr14
+; SI-NEXT: ; implicit-def: $vgpr12
; SI-NEXT: ; implicit-def: $sgpr76
; SI-NEXT: ; implicit-def: $sgpr44
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr50
; SI-NEXT: ; implicit-def: $sgpr27
-; SI-NEXT: ; implicit-def: $vgpr3
+; SI-NEXT: ; implicit-def: $vgpr60
; SI-NEXT: ; implicit-def: $sgpr28
-; SI-NEXT: ; implicit-def: $vgpr59
+; SI-NEXT: ; implicit-def: $vgpr57
; SI-NEXT: ; implicit-def: $sgpr21
-; SI-NEXT: ; implicit-def: $vgpr45
+; SI-NEXT: ; implicit-def: $vgpr55
; SI-NEXT: ; implicit-def: $sgpr22
-; SI-NEXT: ; implicit-def: $vgpr50
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $sgpr56
-; SI-NEXT: ; implicit-def: $vgpr38
+; SI-NEXT: ; implicit-def: $vgpr4
; SI-NEXT: ; implicit-def: $sgpr18
-; SI-NEXT: ; implicit-def: $vgpr32
; SI-NEXT: ; implicit-def: $sgpr91
-; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $sgpr12
; SI-NEXT: ; implicit-def: $sgpr6
; SI-NEXT: ; implicit-def: $sgpr38
@@ -158103,16 +157643,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: ; implicit-def: $sgpr10
; SI-NEXT: ; implicit-def: $sgpr36
; SI-NEXT: ; implicit-def: $sgpr13
-; SI-NEXT: ; implicit-def: $vgpr49
-; SI-NEXT: v_writelane_b32 v62, s4, 46
-; SI-NEXT: v_writelane_b32 v62, s5, 47
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 48
-; SI-NEXT: v_writelane_b32 v62, s5, 49
-; SI-NEXT: ; implicit-def: $sgpr4
-; SI-NEXT: v_writelane_b32 v62, s4, 50
-; SI-NEXT: v_writelane_b32 v62, s5, 51
-; SI-NEXT: ; implicit-def: $sgpr4
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: ; implicit-def: $vgpr42
+; SI-NEXT: ; implicit-def: $vgpr53
; SI-NEXT: v_writelane_b32 v62, s4, 52
; SI-NEXT: v_writelane_b32 v62, s5, 53
; SI-NEXT: ; implicit-def: $sgpr4
@@ -158183,211 +157717,201 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_writelane_b32 v61, s14, 32
; SI-NEXT: v_writelane_b32 v61, s15, 33
; SI-NEXT: v_writelane_b32 v61, s4, 34
-; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: v_writelane_b32 v61, s5, 35
+; SI-NEXT: ; implicit-def: $sgpr14
; SI-NEXT: .LBB91_3: ; %Flow
-; SI-NEXT: v_mov_b32_e32 v13, v16
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: s_and_b64 s[8:9], s[8:9], exec
; SI-NEXT: s_cselect_b32 s5, 1, 0
-; SI-NEXT: v_mov_b32_e32 v2, v19
; SI-NEXT: s_cmp_lg_u32 s5, 1
-; SI-NEXT: v_mov_b32_e32 v54, v38
-; SI-NEXT: v_mov_b32_e32 v38, v50
-; SI-NEXT: v_mov_b32_e32 v50, v45
-; SI-NEXT: v_mov_b32_e32 v45, v3
-; SI-NEXT: v_mov_b32_e32 v3, v10
-; SI-NEXT: v_mov_b32_e32 v10, v14
; SI-NEXT: s_cbranch_scc1 .LBB91_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v15
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s4, v10
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v38
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v10
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s4, v12
; SI-NEXT: s_lshr_b32 s5, s4, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
+; SI-NEXT: v_readfirstlane_b32 s4, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v32
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v6
; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v11
-; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v19, 0x40c00000, v12
+; SI-NEXT: v_readfirstlane_b32 s6, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v31
; SI-NEXT: v_readfirstlane_b32 s5, v19
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v25
; SI-NEXT: s_lshr_b32 s7, s5, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v21
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v13
; SI-NEXT: s_lshr_b64 s[90:91], s[6:7], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v33
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v22
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v10
+; SI-NEXT: v_readfirstlane_b32 s6, v12
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v11
-; SI-NEXT: v_readfirstlane_b32 s12, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
+; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: v_readfirstlane_b32 s12, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v17
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_readfirstlane_b32 s18, v4
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s13, s6, 16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s6, v6
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
; SI-NEXT: s_lshr_b64 s[10:11], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v9
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; SI-NEXT: v_readfirstlane_b32 s18, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v8
+; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
+; SI-NEXT: v_readfirstlane_b32 s8, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v43
-; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v55
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
-; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_readfirstlane_b32 s6, v3
+; SI-NEXT: s_lshr_b32 s19, s6, 16
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v29
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v47
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v54
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v48
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
+; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; SI-NEXT: s_mov_b32 s5, s7
; SI-NEXT: v_writelane_b32 v61, s4, 34
; SI-NEXT: v_writelane_b32 v61, s5, 35
+; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
+; SI-NEXT: s_mov_b32 s17, s34
; SI-NEXT: s_lshr_b64 s[20:21], s[12:13], 16
; SI-NEXT: s_mov_b32 s11, s20
; SI-NEXT: s_mov_b32 s5, s90
+; SI-NEXT: s_lshr_b32 s91, s34, 8
; SI-NEXT: s_lshr_b32 s12, s20, 8
-; SI-NEXT: v_lshrrev_b32_e32 v25, 24, v9
-; SI-NEXT: v_lshrrev_b32_e32 v21, 24, v13
-; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v4
-; SI-NEXT: v_lshrrev_b32_e32 v49, 24, v11
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: v_readfirstlane_b32 s6, v7
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: s_lshr_b64 s[16:17], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s8, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v5
+; SI-NEXT: v_lshrrev_b32_e32 v42, 24, v3
+; SI-NEXT: v_lshrrev_b32_e32 v53, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v33, 24, v19
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_readfirstlane_b32 s6, v4
-; SI-NEXT: s_lshr_b32 s19, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v2
-; SI-NEXT: v_readfirstlane_b32 s22, v3
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
+; SI-NEXT: v_readfirstlane_b32 s22, v4
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
-; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v31
+; SI-NEXT: s_lshr_b64 s[24:25], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v2
-; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
; SI-NEXT: s_lshr_b32 s23, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v6
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v44
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v43
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; SI-NEXT: v_readfirstlane_b32 s8, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v49
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v56
+; SI-NEXT: v_readfirstlane_b32 s28, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v58
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v56
-; SI-NEXT: v_readfirstlane_b32 s6, v6
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
+; SI-NEXT: v_readfirstlane_b32 s6, v4
+; SI-NEXT: v_readfirstlane_b32 s8, v6
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v39
; SI-NEXT: s_lshr_b32 s29, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v7
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v40
-; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; SI-NEXT: s_lshr_b64 s[26:27], s[22:23], 16
-; SI-NEXT: s_mov_b32 s25, s26
-; SI-NEXT: s_lshr_b64 s[34:35], s[18:19], 16
-; SI-NEXT: s_mov_b32 s17, s34
-; SI-NEXT: s_lshr_b32 s18, s26, 8
-; SI-NEXT: s_lshr_b32 s91, s34, 8
-; SI-NEXT: v_lshrrev_b32_e32 v38, 24, v7
-; SI-NEXT: v_lshrrev_b32_e32 v54, 24, v6
-; SI-NEXT: v_lshrrev_b32_e32 v32, 24, v2
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v47
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s28, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v58
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: s_lshr_b64 s[40:41], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v36
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s44, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v46
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
+; SI-NEXT: v_readfirstlane_b32 s44, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v45
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v44
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b64 s[46:47], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v35
-; SI-NEXT: v_readfirstlane_b32 s6, v7
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v36
+; SI-NEXT: v_readfirstlane_b32 s6, v6
+; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b32 s45, s6, 16
; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: v_readfirstlane_b32 s58, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v41
+; SI-NEXT: v_readfirstlane_b32 s58, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v41
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v52
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v40
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b64 s[62:63], s[8:9], 16
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v30
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v35
; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b32 s59, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_readfirstlane_b32 s72, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v53
+; SI-NEXT: v_readfirstlane_b32 s6, v11
+; SI-NEXT: v_readfirstlane_b32 s72, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v52
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v24
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v51
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b64 s[78:79], s[8:9], 16
-; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v51
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v29
-; SI-NEXT: v_readfirstlane_b32 s6, v12
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v14
+; SI-NEXT: v_readfirstlane_b32 s6, v11
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
; SI-NEXT: s_lshr_b32 s73, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v39
-; SI-NEXT: v_readfirstlane_b32 s14, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v37
+; SI-NEXT: v_readfirstlane_b32 s6, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v37
+; SI-NEXT: v_readfirstlane_b32 s14, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v30
; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v10
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v34
+; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v12
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v28
; SI-NEXT: s_lshr_b64 s[30:31], s[8:9], 16
-; SI-NEXT: v_readfirstlane_b32 s6, v15
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s8, v3
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v48
+; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_readfirstlane_b32 s8, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v34
; SI-NEXT: s_lshr_b32 s15, s6, 16
-; SI-NEXT: v_readfirstlane_b32 s6, v10
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s36, v10
+; SI-NEXT: v_readfirstlane_b32 s6, v12
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v7
+; SI-NEXT: v_readfirstlane_b32 s36, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v20
; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
; SI-NEXT: s_lshr_b64 s[76:77], s[8:9], 16
-; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v23
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; SI-NEXT: v_readfirstlane_b32 s6, v3
+; SI-NEXT: v_readfirstlane_b32 s8, v12
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v22
+; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v7
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
; SI-NEXT: s_lshr_b32 s37, s6, 16
+; SI-NEXT: v_readfirstlane_b32 s6, v15
+; SI-NEXT: s_lshr_b32 s9, s6, 16
+; SI-NEXT: s_lshr_b64 s[94:95], s[8:9], 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b64 s[38:39], s[36:37], 16
; SI-NEXT: s_mov_b32 s77, s38
; SI-NEXT: s_lshr_b64 s[96:97], s[14:15], 16
@@ -158395,84 +157919,85 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b64 s[92:93], s[72:73], 16
; SI-NEXT: s_mov_b32 s79, s92
; SI-NEXT: s_lshr_b64 s[60:61], s[58:59], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s8, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_readfirstlane_b32 s6, v14
-; SI-NEXT: s_lshr_b32 s9, s6, 16
-; SI-NEXT: s_lshr_b64 s[94:95], s[8:9], 16
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b32 s63, s60
; SI-NEXT: s_lshr_b64 s[52:53], s[44:45], 16
; SI-NEXT: s_mov_b32 s47, s52
; SI-NEXT: s_lshr_b64 s[56:57], s[28:29], 16
; SI-NEXT: s_mov_b32 s41, s56
+; SI-NEXT: s_lshr_b64 s[26:27], s[22:23], 16
+; SI-NEXT: s_mov_b32 s25, s26
; SI-NEXT: s_lshr_b32 s44, s38, 8
; SI-NEXT: s_lshr_b32 s27, s96, 8
; SI-NEXT: s_lshr_b32 s28, s92, 8
; SI-NEXT: s_lshr_b32 s21, s60, 8
; SI-NEXT: s_lshr_b32 s22, s52, 8
+; SI-NEXT: s_lshr_b32 s18, s26, 8
; SI-NEXT: s_mov_b32 s36, s20
; SI-NEXT: s_mov_b32 s14, s90
-; SI-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; SI-NEXT: v_lshrrev_b32_e32 v45, 24, v15
-; SI-NEXT: v_lshrrev_b32_e32 v59, 24, v12
-; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v8
+; SI-NEXT: v_lshrrev_b32_e32 v50, 24, v7
+; SI-NEXT: v_lshrrev_b32_e32 v60, 24, v14
+; SI-NEXT: v_lshrrev_b32_e32 v57, 24, v11
+; SI-NEXT: v_lshrrev_b32_e32 v55, 24, v8
+; SI-NEXT: v_lshrrev_b32_e32 v11, 24, v6
+; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v4
+; SI-NEXT: v_lshrrev_b32_e32 v46, 24, v2
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s88, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v60
-; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; SI-NEXT: v_readfirstlane_b32 s6, v10
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v12
+; SI-NEXT: v_readfirstlane_b32 s88, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v59
+; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; SI-NEXT: v_readfirstlane_b32 s6, v12
; SI-NEXT: s_lshr_b32 s89, s6, 16
; SI-NEXT: s_lshr_b64 s[80:81], s[88:89], 16
; SI-NEXT: s_mov_b32 s95, s80
; SI-NEXT: s_lshr_b32 s93, s80, 8
-; SI-NEXT: v_lshrrev_b32_e32 v10, 24, v10
+; SI-NEXT: v_lshrrev_b32_e32 v12, 24, v12
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_readfirstlane_b32 s8, v14
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_readfirstlane_b32 s8, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s6, v16
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[48:49], s[8:9], 16
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v14
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v14
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v15
; SI-NEXT: v_readfirstlane_b32 s42, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; SI-NEXT: v_readfirstlane_b32 s6, v14
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; SI-NEXT: v_readfirstlane_b32 s6, v15
; SI-NEXT: s_lshr_b32 s43, s6, 16
; SI-NEXT: s_lshr_b64 vcc, s[42:43], 16
; SI-NEXT: s_mov_b32 s49, vcc_lo
; SI-NEXT: s_lshr_b32 s58, vcc_lo, 8
; SI-NEXT: s_mov_b32 s42, s26
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v15, 24, v15
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; SI-NEXT: v_readfirstlane_b32 s8, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; SI-NEXT: v_lshrrev_b32_e32 v18, 24, v13
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s74, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[64:65], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v13
@@ -158482,28 +158007,28 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_lshr_b32 s57, s54, 8
; SI-NEXT: s_mov_b32 s74, s60
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; SI-NEXT: v_lshrrev_b32_e32 v26, 24, v5
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
+; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; SI-NEXT: v_lshrrev_b32_e32 v23, 24, v9
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v18, 24, v1
+; SI-NEXT: v_lshrrev_b32_e32 v27, 24, v1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s66, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[70:71], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v9
@@ -158517,16 +158042,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s50, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_readfirstlane_b32 s6, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_lshr_b32 s9, s6, 16
; SI-NEXT: s_lshr_b64 s[84:85], s[8:9], 16
; SI-NEXT: v_readfirstlane_b32 s6, v5
@@ -158538,7 +158063,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s8, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
@@ -158547,10 +158072,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readfirstlane_b32 s6, v1
; SI-NEXT: s_lshr_b32 s87, s6, 16
; SI-NEXT: s_lshr_b64 s[98:99], s[8:9], 16
-; SI-NEXT: v_lshrrev_b32_e32 v17, 24, v14
-; SI-NEXT: v_lshrrev_b32_e32 v1, 24, v19
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
; SI-NEXT: v_readfirstlane_b32 s86, v16
@@ -158708,21 +158230,18 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_mov_b32 s86, s92
; SI-NEXT: s_lshr_b32 s6, s90, 8
; SI-NEXT: .LBB91_5: ; %end
-; SI-NEXT: v_readlane_b32 s8, v62, 0
+; SI-NEXT: v_readlane_b32 s7, v62, 0
; SI-NEXT: s_and_b32 s5, s98, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 1
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 2
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 2
-; SI-NEXT: v_readlane_b32 s9, v62, 3
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s98, v62, 4
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 4
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s98, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s5
; SI-NEXT: s_and_b32 s5, s96, 0xff
; SI-NEXT: s_lshl_b32 s8, s88, 8
@@ -158730,22 +158249,20 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_and_b32 s8, s87, 0xff
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_waitcnt vmcnt(1) expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v18
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v27
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 6
+; SI-NEXT: v_readlane_b32 s7, v62, 6
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s84, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 7
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 8
-; SI-NEXT: v_readlane_b32 s9, v62, 9
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s84, v62, 10
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 10
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s84, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158762,18 +158279,16 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v26
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 12
+; SI-NEXT: v_readlane_b32 s7, v62, 12
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s70, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 13
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 14
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 14
-; SI-NEXT: v_readlane_b32 s9, v62, 15
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 16
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 16
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158787,22 +158302,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v25
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v23
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 18
+; SI-NEXT: v_readlane_b32 s7, v62, 18
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s64, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 19
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 17
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 20
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 20
-; SI-NEXT: v_readlane_b32 s9, v62, 21
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 22
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 22
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158816,22 +158328,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v21
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v18
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 24
+; SI-NEXT: v_readlane_b32 s7, v62, 24
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s48, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 25
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 23
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 26
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 26
-; SI-NEXT: v_readlane_b32 s9, v62, 27
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 28
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 28
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158845,22 +158354,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v17
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v15
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 30
+; SI-NEXT: v_readlane_b32 s7, v62, 30
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s94, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 31
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 29
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 32
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 32
-; SI-NEXT: v_readlane_b32 s9, v62, 33
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 34
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 34
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158874,22 +158380,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v10
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v12
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 36
+; SI-NEXT: v_readlane_b32 s7, v62, 36
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s76, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 37
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 35
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 38
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 38
-; SI-NEXT: v_readlane_b32 s9, v62, 39
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 40
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 40
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158903,52 +158406,46 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v50
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 42
+; SI-NEXT: v_readlane_b32 s7, v62, 42
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s30, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 43
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s61, v62, 41
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 44
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 44
-; SI-NEXT: v_readlane_b32 s9, v62, 45
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s60, v62, 46
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 46
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s60, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 48
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: s_and_b32 s5, s8, 0xff
+; SI-NEXT: v_readlane_b32 s5, v62, 48
+; SI-NEXT: s_and_b32 s5, s5, 0xff
; SI-NEXT: s_lshl_b32 s8, s27, 8
; SI-NEXT: s_or_b32 s5, s5, s8
; SI-NEXT: s_and_b32 s8, s15, 0xff
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
-; SI-NEXT: v_readlane_b32 s9, v62, 49
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v45
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v60
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 50
+; SI-NEXT: v_readlane_b32 s7, v62, 50
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s78, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 51
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 52
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 52
-; SI-NEXT: v_readlane_b32 s9, v62, 53
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 54
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 54
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158962,22 +158459,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v59
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v57
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 56
+; SI-NEXT: v_readlane_b32 s7, v62, 56
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s62, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 57
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s27, v62, 55
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v62, 58
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v62, 58
-; SI-NEXT: v_readlane_b32 s9, v62, 59
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s26, v62, 60
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v62, 60
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s26, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -158991,21 +158485,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:72
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v50
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v55
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v62, 62
+; SI-NEXT: v_readlane_b32 s7, v62, 62
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s46, 0xff
-; SI-NEXT: v_readlane_b32 s9, v62, 63
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 0
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 0
-; SI-NEXT: v_readlane_b32 s9, v61, 1
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 2
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 2
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -159019,22 +158511,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v38
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v11
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 4
+; SI-NEXT: v_readlane_b32 s7, v61, 4
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s40, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 5
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s21, v61, 3
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 6
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 6
-; SI-NEXT: v_readlane_b32 s9, v61, 7
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 8
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 8
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -159048,22 +158537,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v54
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v4
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 10
+; SI-NEXT: v_readlane_b32 s7, v61, 10
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s24, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 11
-; SI-NEXT: s_lshl_b32 s8, s8, 8
-; SI-NEXT: v_readlane_b32 s21, v61, 9
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 12
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 12
-; SI-NEXT: v_readlane_b32 s9, v61, 13
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s20, v61, 14
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 14
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s20, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -159077,21 +158563,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v32
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v46
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 16
+; SI-NEXT: v_readlane_b32 s7, v61, 16
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s16, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 17
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 18
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 18
-; SI-NEXT: v_readlane_b32 s9, v61, 19
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s16, v61, 20
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 20
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s16, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -159105,21 +158589,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v27
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v42
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 22
+; SI-NEXT: v_readlane_b32 s7, v61, 22
; SI-NEXT: v_or_b32_e32 v1, s5, v1
; SI-NEXT: s_and_b32 s5, s10, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 23
-; SI-NEXT: s_lshl_b32 s8, s8, 8
+; SI-NEXT: s_lshl_b32 s8, s7, 8
+; SI-NEXT: v_readlane_b32 s7, v61, 24
; SI-NEXT: s_or_b32 s5, s5, s8
-; SI-NEXT: v_readlane_b32 s8, v61, 24
-; SI-NEXT: v_readlane_b32 s9, v61, 25
-; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v61, 26
+; SI-NEXT: s_and_b32 s8, s7, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 26
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s7, 24
; SI-NEXT: s_and_b32 s5, s5, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s5, s5, s8
@@ -159133,47 +158615,52 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
; SI-NEXT: s_lshl_b32 s8, s8, 16
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v49
-; SI-NEXT: v_or_b32_e32 v1, s8, v1
-; SI-NEXT: v_readlane_b32 s8, v61, 28
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v53
; SI-NEXT: s_and_b32 s5, s5, 0xffff
-; SI-NEXT: v_readlane_b32 s9, v61, 29
+; SI-NEXT: v_or_b32_e32 v1, s8, v1
; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: s_lshl_b32 s5, s8, 8
-; SI-NEXT: v_readlane_b32 s8, v61, 30
+; SI-NEXT: v_readlane_b32 s5, v61, 28
; SI-NEXT: s_and_b32 s4, s4, 0xff
-; SI-NEXT: v_readlane_b32 s9, v61, 31
+; SI-NEXT: s_lshl_b32 s5, s5, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s8, 0xff
-; SI-NEXT: v_readlane_b32 s8, v61, 32
+; SI-NEXT: v_readlane_b32 s5, v61, 30
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: v_readlane_b32 s7, v61, 32
; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_lshl_b32 s8, s8, 24
+; SI-NEXT: s_lshl_b32 s8, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s5, s8, s5
; SI-NEXT: s_or_b32 s4, s4, s5
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; SI-NEXT: s_lshl_b32 s5, s6, 8
-; SI-NEXT: v_readlane_b32 s6, v61, 34
; SI-NEXT: s_and_b32 s4, s14, 0xff
-; SI-NEXT: v_readlane_b32 s7, v61, 35
+; SI-NEXT: s_lshl_b32 s5, s6, 8
; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s7, 0xff
+; SI-NEXT: v_readlane_b32 s5, v61, 35
+; SI-NEXT: s_and_b32 s5, s5, 0xff
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; SI-NEXT: s_lshl_b32 s5, s5, 16
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v33
; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: v_readlane_b32 s99, v62, 5
-; SI-NEXT: v_readlane_b32 s85, v62, 11
+; SI-NEXT: v_or_b32_e32 v1, s5, v1
+; SI-NEXT: v_or_b32_e32 v1, s4, v1
+; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
+; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: v_readlane_b32 s30, v63, 34
-; SI-NEXT: v_readlane_b32 s61, v62, 47
-; SI-NEXT: v_readlane_b32 s27, v62, 61
-; SI-NEXT: v_readlane_b32 s21, v61, 15
-; SI-NEXT: v_readlane_b32 s17, v61, 21
-; SI-NEXT: v_readlane_b32 s11, v61, 27
-; SI-NEXT: v_readlane_b32 s9, v61, 33
; SI-NEXT: v_readlane_b32 s31, v63, 35
; SI-NEXT: v_readlane_b32 s99, v63, 33
; SI-NEXT: v_readlane_b32 s98, v63, 32
@@ -159209,28 +158696,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; SI-NEXT: v_or_b32_e32 v1, s5, v1
-; SI-NEXT: v_or_b32_e32 v1, s4, v1
-; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -160305,8 +159774,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v59
-; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v60
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
@@ -160315,8 +159782,41 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v60
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; VI-NEXT: s_waitcnt vmcnt(6)
-; VI-NEXT: v_mov_b32_e32 v2, v58
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v41, v54
+; VI-NEXT: v_lshrrev_b64 v[42:43], 24, v[40:41]
+; VI-NEXT: v_mov_b32_e32 v39, v37
+; VI-NEXT: v_mov_b32_e32 v30, v28
+; VI-NEXT: v_mov_b32_e32 v33, v31
+; VI-NEXT: v_mov_b32_e32 v36, v34
+; VI-NEXT: v_mov_b32_e32 v50, v48
+; VI-NEXT: v_lshrrev_b64 v[43:44], 24, v[38:39]
+; VI-NEXT: v_lshrrev_b64 v[6:7], 24, v[49:50]
+; VI-NEXT: v_lshrrev_b64 v[44:45], 24, v[35:36]
+; VI-NEXT: v_lshrrev_b64 v[12:13], 24, v[32:33]
+; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[29:30]
+; VI-NEXT: v_lshrrev_b64 v[4:5], 24, v[17:18]
+; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v52
+; VI-NEXT: v_lshrrev_b32_e32 v7, 8, v49
+; VI-NEXT: v_lshrrev_b32_e32 v11, 8, v37
+; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v38
+; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
+; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v34
+; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v35
+; VI-NEXT: v_lshrrev_b32_e32 v20, 8, v35
+; VI-NEXT: v_lshrrev_b32_e32 v27, 8, v31
+; VI-NEXT: v_lshrrev_b32_e32 v25, 16, v32
+; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v32
+; VI-NEXT: v_lshrrev_b32_e32 v50, 8, v28
+; VI-NEXT: v_lshrrev_b32_e32 v23, 16, v29
+; VI-NEXT: v_lshrrev_b32_e32 v53, 8, v29
+; VI-NEXT: v_mov_b32_e32 v60, v8
+; VI-NEXT: v_lshrrev_b32_e32 v39, 8, v8
+; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v56
+; VI-NEXT: v_lshrrev_b32_e32 v47, 8, v56
+; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
+; VI-NEXT: v_mov_b32_e32 v58, v17
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_lshrrev_b32_e32 v16, 24, v2
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v16, 16, v2
@@ -160360,50 +159860,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v15
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; VI-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v41, v54
-; VI-NEXT: v_mov_b32_e32 v39, v37
-; VI-NEXT: v_lshrrev_b64 v[42:43], 24, v[40:41]
-; VI-NEXT: v_mov_b32_e32 v30, v28
-; VI-NEXT: v_mov_b32_e32 v33, v31
-; VI-NEXT: v_mov_b32_e32 v36, v34
-; VI-NEXT: v_mov_b32_e32 v50, v48
-; VI-NEXT: v_lshrrev_b64 v[43:44], 24, v[38:39]
-; VI-NEXT: v_lshrrev_b64 v[6:7], 24, v[49:50]
-; VI-NEXT: v_lshrrev_b64 v[44:45], 24, v[35:36]
-; VI-NEXT: v_lshrrev_b64 v[12:13], 24, v[32:33]
-; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[29:30]
-; VI-NEXT: v_lshrrev_b64 v[4:5], 24, v[17:18]
-; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v52
-; VI-NEXT: v_lshrrev_b32_e32 v7, 8, v49
-; VI-NEXT: v_lshrrev_b32_e32 v11, 8, v37
-; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v38
-; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v38
-; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v34
-; VI-NEXT: v_lshrrev_b32_e32 v18, 16, v35
-; VI-NEXT: v_lshrrev_b32_e32 v20, 8, v35
-; VI-NEXT: v_lshrrev_b32_e32 v27, 8, v31
-; VI-NEXT: v_lshrrev_b32_e32 v25, 16, v32
-; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v32
-; VI-NEXT: v_lshrrev_b32_e32 v50, 8, v28
-; VI-NEXT: v_lshrrev_b32_e32 v23, 16, v29
-; VI-NEXT: v_lshrrev_b32_e32 v53, 8, v29
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v16, 8, v24
-; VI-NEXT: v_mov_b32_e32 v60, v8
-; VI-NEXT: v_lshrrev_b32_e32 v39, 8, v8
; VI-NEXT: v_lshrrev_b32_e32 v21, 24, v56
-; VI-NEXT: v_lshrrev_b32_e32 v55, 16, v56
-; VI-NEXT: v_lshrrev_b32_e32 v47, 8, v56
-; VI-NEXT: v_lshrrev_b32_e32 v45, 16, v17
-; VI-NEXT: s_waitcnt vmcnt(5)
-; VI-NEXT: v_mov_b32_e32 v2, v14
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshrrev_b32_e32 v10, 24, v2
; VI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v10, 16, v2
@@ -160416,29 +159878,27 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v2, 8, v26
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v10, v33
; VI-NEXT: v_lshrrev_b32_e32 v26, 16, v24
-; VI-NEXT: s_waitcnt vmcnt(10)
-; VI-NEXT: v_mov_b32_e32 v59, v58
; VI-NEXT: v_lshrrev_b32_e32 v33, 16, v8
; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v49
-; VI-NEXT: s_waitcnt vmcnt(8)
-; VI-NEXT: v_mov_b32_e32 v2, v14
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshrrev_b32_e32 v14, 24, v2
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v14, 16, v2
; VI-NEXT: v_mov_b32_e32 v41, v2
; VI-NEXT: v_lshrrev_b32_e32 v22, 8, v2
-; VI-NEXT: v_mov_b32_e32 v2, v58
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v14, v24
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_lshrrev_b32_e32 v61, 24, v2
-; VI-NEXT: s_waitcnt vmcnt(9)
; VI-NEXT: v_lshrrev_b32_e32 v15, 16, v2
+; VI-NEXT: v_mov_b32_e32 v59, v2
; VI-NEXT: v_lshrrev_b32_e32 v36, 8, v2
; VI-NEXT: v_mov_b32_e32 v2, v56
-; VI-NEXT: v_mov_b32_e32 v58, v17
; VI-NEXT: v_lshrrev_b32_e32 v56, 8, v17
; VI-NEXT: s_branch .LBB91_6
; VI-NEXT: .LBB91_5:
@@ -160726,13 +160186,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_perm_b32 v28, v32, v30, s4
; VI-NEXT: v_perm_b32 v24, v34, v9, s4
; VI-NEXT: v_perm_b32 v20, v35, v20, s4
-; VI-NEXT: v_perm_b32 v13, v38, v13, s4
; VI-NEXT: v_perm_b32 v11, v37, v11, s4
; VI-NEXT: v_perm_b32 v7, v49, v7, s4
; VI-NEXT: v_perm_b32 v49, v2, v47, s4
; VI-NEXT: v_perm_b32 v29, v29, v53, s4
; VI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
; VI-NEXT: v_or_b32_e32 v19, v29, v19
+; VI-NEXT: v_perm_b32 v13, v38, v13, s4
; VI-NEXT: v_perm_b32 v18, v18, v44, s4
; VI-NEXT: v_perm_b32 v10, v52, v10, s4
; VI-NEXT: v_perm_b32 v1, v33, v1, s4
@@ -160776,11 +160236,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_readlane_b32 s36, v63, 2
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v8, v48, v5, s4
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v48, v58, v56, s4
; VI-NEXT: v_or_b32_e32 v4, v48, v4
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v26, v26, v2, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v27, v5, v3, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
@@ -160799,18 +160262,13 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v35, v57, v3, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v37, v3, v37, s4
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v38, v46, v3, s4
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v37, v3, v5, s4
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen
-; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v26, v26, v2, s4
+; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: v_perm_b32 v38, v46, v3, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -160909,12 +160367,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v22, v1
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
@@ -160922,7 +160378,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -160934,12 +160389,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
@@ -160947,7 +160400,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -160959,12 +160411,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
@@ -160972,7 +160422,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -160984,12 +160433,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
@@ -160997,7 +160444,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -161009,12 +160455,10 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v3, v2, s4
@@ -161022,7 +160466,6 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
@@ -162704,15 +162147,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
-; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
@@ -162725,15 +162167,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
-; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
@@ -162746,15 +162187,14 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
-; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -162774,8 +162214,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
@@ -162799,8 +162238,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v2, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179274,15 +178712,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
; VI-NEXT: v_perm_b32 v4, v52, v5, s4
-; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v2, v2, v30, s4
; VI-NEXT: v_perm_b32 v29, v39, v43, s4
; VI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; VI-NEXT: v_perm_b32 v17, v17, v60, s4
; VI-NEXT: v_perm_b32 v18, v18, v62, s4
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(3)
+; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_perm_b32 v1, v1, v3, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; VI-NEXT: v_or_b32_e32 v1, v1, v29
@@ -179301,8 +178738,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v8, v8, v13, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v39, v13, v14, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -179312,8 +178748,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v10, v10, v13, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v48, v13, v14, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -179323,8 +178758,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v12, v12, v13, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v49, v13, v14, s4
; VI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
@@ -179335,10 +178769,9 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v14, v14, v15, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v50, v15, v21, s4
+; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v50, v15, v16, s4
; VI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
@@ -179452,8 +178885,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v1, v2, v1, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v2, v4, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179467,12 +178899,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v25, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179486,12 +178918,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v27, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179512,8 +178944,7 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v1, v2, v1, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v2, v4, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179527,12 +178958,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v53, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -179546,12 +178977,12 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v23, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -180324,9 +179755,8 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v1, v21, v47, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v46, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -180340,14 +179770,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -180360,14 +179790,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -180380,14 +179810,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
@@ -180400,14 +179830,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
@@ -180420,14 +179850,14 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -182436,7 +181866,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f32_f16_e32 v13, s21
; SI-NEXT: v_readlane_b32 s21, v36, 1
; SI-NEXT: v_add_f32_e32 v14, 0x38000000, v14
-; SI-NEXT: v_cvt_f16_f32_e32 v19, v14
+; SI-NEXT: v_cvt_f16_f32_e32 v18, v14
; SI-NEXT: v_add_f32_e32 v13, 0x38000000, v13
; SI-NEXT: v_cvt_f32_f16_e32 v14, s21
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
@@ -182454,7 +181884,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_cvt_f16_f32_e32 v13, v13
; SI-NEXT: s_lshr_b64 s[50:51], s[12:13], 16
; SI-NEXT: s_or_b32 s62, s22, s20
-; SI-NEXT: v_readfirstlane_b32 s20, v19
+; SI-NEXT: v_readfirstlane_b32 s20, v18
; SI-NEXT: v_writelane_b32 v37, s50, 15
; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: v_writelane_b32 v37, s51, 16
@@ -182567,15 +181997,15 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshr_b32 s27, s45, 8
; SI-NEXT: s_lshr_b32 s26, s43, 8
; SI-NEXT: s_lshr_b32 s37, s41, 8
-; SI-NEXT: s_lshr_b32 s19, s15, 8
-; SI-NEXT: s_lshr_b32 s35, s13, 8
-; SI-NEXT: s_lshr_b32 s31, s11, 8
+; SI-NEXT: s_lshr_b32 s35, s15, 8
+; SI-NEXT: s_lshr_b32 s31, s13, 8
+; SI-NEXT: s_lshr_b32 s23, s11, 8
; SI-NEXT: s_lshr_b32 vcc_hi, s9, 8
; SI-NEXT: s_lshr_b32 s95, s7, 8
-; SI-NEXT: s_lshr_b32 s23, s5, 8
+; SI-NEXT: s_lshr_b32 s19, s5, 8
; SI-NEXT: v_bfe_u32 v32, v26, 8, 8
; SI-NEXT: v_bfe_u32 v31, v22, 8, 8
-; SI-NEXT: v_bfe_u32 v30, v19, 8, 8
+; SI-NEXT: v_bfe_u32 v30, v18, 8, 8
; SI-NEXT: v_bfe_u32 v29, v15, 8, 8
; SI-NEXT: v_bfe_u32 v28, v12, 8, 8
; SI-NEXT: v_bfe_u32 v27, v11, 8, 8
@@ -182584,7 +182014,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_bfe_u32 v23, v8, 8, 8
; SI-NEXT: v_bfe_u32 v21, v7, 8, 8
; SI-NEXT: v_bfe_u32 v20, v6, 8, 8
-; SI-NEXT: v_bfe_u32 v18, v5, 8, 8
+; SI-NEXT: v_bfe_u32 v19, v5, 8, 8
; SI-NEXT: v_bfe_u32 v17, v4, 8, 8
; SI-NEXT: v_bfe_u32 v16, v3, 8, 8
; SI-NEXT: v_bfe_u32 v14, v2, 8, 8
@@ -182617,116 +182047,176 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s16, v37, 48
; SI-NEXT: v_mov_b32_e32 v30, s16
; SI-NEXT: v_readlane_b32 s16, v37, 49
-; SI-NEXT: v_mov_b32_e32 v7, vcc_hi
; SI-NEXT: v_mov_b32_e32 v31, s16
; SI-NEXT: v_readlane_b32 s16, v37, 50
-; SI-NEXT: v_readlane_b32 s94, v36, 21
-; SI-NEXT: v_readlane_b32 vcc_lo, v36, 23
; SI-NEXT: v_mov_b32_e32 v32, s16
-; SI-NEXT: v_readlane_b32 s16, v37, 52
-; SI-NEXT: v_readlane_b32 s17, v37, 53
-; SI-NEXT: v_readlane_b32 s95, v36, 22
-; SI-NEXT: v_readlane_b32 vcc_hi, v36, 24
-; SI-NEXT: v_mov_b32_e32 v6, s35
-; SI-NEXT: v_mov_b32_e32 v14, s37
-; SI-NEXT: s_mov_b32 s95, s16
-; SI-NEXT: s_mov_b32 vcc_hi, s17
-; SI-NEXT: v_readlane_b32 s30, v36, 19
-; SI-NEXT: v_readlane_b32 s16, v36, 15
-; SI-NEXT: v_readlane_b32 s34, v36, 17
-; SI-NEXT: v_readlane_b32 s36, v36, 13
-; SI-NEXT: v_readlane_b32 s22, v36, 37
-; SI-NEXT: v_mov_b32_e32 v8, s93
-; SI-NEXT: v_mov_b32_e32 v9, s90
+; SI-NEXT: v_readlane_b32 s16, v36, 33
+; SI-NEXT: v_mov_b32_e32 v18, s76
+; SI-NEXT: s_mov_b32 s76, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 31
+; SI-NEXT: v_mov_b32_e32 v12, s78
+; SI-NEXT: s_mov_b32 s78, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 27
; SI-NEXT: v_mov_b32_e32 v10, s88
+; SI-NEXT: s_mov_b32 s88, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 29
+; SI-NEXT: v_mov_b32_e32 v9, s90
+; SI-NEXT: s_mov_b32 s90, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 25
+; SI-NEXT: s_mov_b32 s92, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 21
+; SI-NEXT: s_mov_b32 s94, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 23
+; SI-NEXT: s_mov_b32 vcc_lo, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 19
+; SI-NEXT: v_readlane_b32 s17, v37, 39
+; SI-NEXT: s_mov_b32 s30, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 15
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, vcc_hi
; SI-NEXT: v_mov_b32_e32 v11, s79
-; SI-NEXT: v_mov_b32_e32 v12, s78
; SI-NEXT: v_mov_b32_e32 v15, s77
-; SI-NEXT: v_mov_b32_e32 v19, s76
-; SI-NEXT: v_readlane_b32 s24, v37, 51
-; SI-NEXT: v_readlane_b32 s18, v37, 54
-; SI-NEXT: v_readlane_b32 s20, v37, 55
-; SI-NEXT: v_readlane_b32 s21, v37, 57
-; SI-NEXT: v_readlane_b32 s76, v36, 33
-; SI-NEXT: v_readlane_b32 s78, v36, 31
-; SI-NEXT: v_readlane_b32 s88, v36, 27
-; SI-NEXT: v_readlane_b32 s90, v36, 29
-; SI-NEXT: v_readlane_b32 s92, v36, 25
-; SI-NEXT: v_readlane_b32 s31, v36, 20
-; SI-NEXT: v_readlane_b32 s17, v36, 16
-; SI-NEXT: v_readlane_b32 s35, v36, 18
-; SI-NEXT: v_readlane_b32 s37, v36, 14
-; SI-NEXT: v_readlane_b32 s23, v36, 38
-; SI-NEXT: v_mov_b32_e32 v1, s70
-; SI-NEXT: v_mov_b32_e32 v2, s66
-; SI-NEXT: v_mov_b32_e32 v3, s54
-; SI-NEXT: v_mov_b32_e32 v4, s50
-; SI-NEXT: v_mov_b32_e32 v5, s38
-; SI-NEXT: v_mov_b32_e32 v22, s99
-; SI-NEXT: v_mov_b32_e32 v26, s98
-; SI-NEXT: v_mov_b32_e32 v13, s82
-; SI-NEXT: v_mov_b32_e32 v16, s81
-; SI-NEXT: v_mov_b32_e32 v17, s87
-; SI-NEXT: v_mov_b32_e32 v18, s86
-; SI-NEXT: v_mov_b32_e32 v20, s83
-; SI-NEXT: v_mov_b32_e32 v21, s48
-; SI-NEXT: v_mov_b32_e32 v23, s80
-; SI-NEXT: v_readlane_b32 s77, v36, 34
-; SI-NEXT: v_readlane_b32 s79, v36, 32
-; SI-NEXT: v_readlane_b32 s89, v36, 28
-; SI-NEXT: v_readlane_b32 s91, v36, 30
-; SI-NEXT: v_readlane_b32 s93, v36, 26
-; SI-NEXT: s_mov_b32 s31, s18
-; SI-NEXT: s_mov_b32 s18, s16
-; SI-NEXT: s_mov_b32 s35, s20
-; SI-NEXT: s_mov_b32 s37, s21
-; SI-NEXT: v_readlane_b32 s16, v36, 11
-; SI-NEXT: v_readlane_b32 s50, v37, 37
-; SI-NEXT: v_readlane_b32 s96, v36, 45
-; SI-NEXT: v_readlane_b32 s84, v36, 43
-; SI-NEXT: v_readlane_b32 s54, v36, 41
-; SI-NEXT: v_readlane_b32 s64, v36, 39
-; SI-NEXT: s_mov_b32 s23, s24
-; SI-NEXT: v_readlane_b32 s86, v36, 35
-; SI-NEXT: v_readlane_b32 s98, v37, 11
-; SI-NEXT: v_readlane_b32 s20, v37, 9
-; SI-NEXT: v_readlane_b32 s48, v37, 7
-; SI-NEXT: v_readlane_b32 s38, v37, 5
-; SI-NEXT: v_readlane_b32 s68, v37, 3
-; SI-NEXT: v_readlane_b32 s24, v37, 1
-; SI-NEXT: v_readlane_b32 s52, v36, 63
-; SI-NEXT: v_readlane_b32 s80, v36, 61
-; SI-NEXT: v_readlane_b32 s66, v36, 59
-; SI-NEXT: v_readlane_b32 s82, v36, 55
-; SI-NEXT: v_readlane_b32 s70, v36, 53
-; SI-NEXT: v_readlane_b32 s19, v37, 56
+; SI-NEXT: v_mov_b32_e32 v14, s37
+; SI-NEXT: v_readlane_b32 s19, v37, 51
+; SI-NEXT: v_readlane_b32 s95, v37, 52
+; SI-NEXT: v_readlane_b32 vcc_hi, v37, 53
+; SI-NEXT: v_readlane_b32 s23, v37, 54
+; SI-NEXT: v_readlane_b32 s31, v37, 55
+; SI-NEXT: v_readlane_b32 s35, v37, 56
+; SI-NEXT: v_readlane_b32 s37, v37, 57
; SI-NEXT: v_readlane_b32 s26, v37, 58
; SI-NEXT: v_readlane_b32 s27, v37, 59
; SI-NEXT: v_readlane_b32 s28, v37, 60
; SI-NEXT: v_readlane_b32 s29, v37, 61
+; SI-NEXT: s_mov_b32 s18, s16
; SI-NEXT: v_readlane_b32 s77, v37, 62
-; SI-NEXT: v_readlane_b32 s17, v36, 12
+; SI-NEXT: v_readlane_b32 s16, v36, 17
; SI-NEXT: v_readlane_b32 s79, v37, 63
+; SI-NEXT: v_writelane_b32 v37, s20, 39
+; SI-NEXT: v_readlane_b32 s17, v36, 57
+; SI-NEXT: s_mov_b32 s34, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 13
+; SI-NEXT: v_writelane_b32 v37, s21, 40
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: s_mov_b32 s36, s16
+; SI-NEXT: v_readlane_b32 s16, v36, 11
+; SI-NEXT: v_writelane_b32 v36, s20, 57
+; SI-NEXT: v_writelane_b32 v36, s21, 58
+; SI-NEXT: v_readlane_b32 s17, v37, 37
+; SI-NEXT: v_mov_b32_e32 v4, s50
+; SI-NEXT: s_mov_b32 s50, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 51
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v36, s20, 51
+; SI-NEXT: v_readlane_b32 s17, v37, 35
+; SI-NEXT: v_writelane_b32 v36, s21, 52
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 35
+; SI-NEXT: v_writelane_b32 v37, s21, 36
+; SI-NEXT: v_readlane_b32 s17, v37, 33
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 33
+; SI-NEXT: v_readlane_b32 s17, v36, 49
+; SI-NEXT: v_writelane_b32 v37, s21, 34
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v36, s20, 49
+; SI-NEXT: v_readlane_b32 s17, v37, 31
+; SI-NEXT: v_writelane_b32 v36, s21, 50
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 31
+; SI-NEXT: v_readlane_b32 s17, v36, 47
+; SI-NEXT: v_writelane_b32 v37, s21, 32
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v36, s20, 47
+; SI-NEXT: v_readlane_b32 s17, v37, 29
+; SI-NEXT: v_writelane_b32 v36, s21, 48
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 29
+; SI-NEXT: v_writelane_b32 v37, s21, 30
+; SI-NEXT: v_readlane_b32 s17, v37, 27
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 27
+; SI-NEXT: v_writelane_b32 v37, s21, 28
+; SI-NEXT: v_readlane_b32 s17, v37, 25
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 25
+; SI-NEXT: v_writelane_b32 v37, s21, 26
+; SI-NEXT: v_readlane_b32 s17, v36, 45
+; SI-NEXT: s_mov_b32 s96, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 23
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 23
+; SI-NEXT: v_writelane_b32 v37, s21, 24
+; SI-NEXT: v_readlane_b32 s17, v36, 43
+; SI-NEXT: s_mov_b32 s84, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 21
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 21
+; SI-NEXT: v_writelane_b32 v37, s21, 22
+; SI-NEXT: v_readlane_b32 s17, v37, 19
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 19
+; SI-NEXT: v_readlane_b32 s17, v36, 41
+; SI-NEXT: v_mov_b32_e32 v3, s54
+; SI-NEXT: v_writelane_b32 v37, s21, 20
+; SI-NEXT: s_mov_b32 s54, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 39
+; SI-NEXT: s_mov_b32 s64, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 17
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 17
+; SI-NEXT: v_writelane_b32 v37, s21, 18
+; SI-NEXT: v_readlane_b32 s17, v36, 37
+; SI-NEXT: s_mov_b32 s22, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 15
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 15
+; SI-NEXT: v_writelane_b32 v37, s21, 16
+; SI-NEXT: v_readlane_b32 s17, v37, 13
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_writelane_b32 v37, s20, 13
+; SI-NEXT: v_writelane_b32 v37, s21, 14
+; SI-NEXT: v_readlane_b32 s17, v36, 35
+; SI-NEXT: v_mov_b32_e32 v19, s86
+; SI-NEXT: s_mov_b32 s86, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 11
+; SI-NEXT: v_mov_b32_e32 v26, s98
+; SI-NEXT: s_mov_b32 s98, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 9
+; SI-NEXT: s_mov_b32 s20, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 7
+; SI-NEXT: v_mov_b32_e32 v21, s48
+; SI-NEXT: s_mov_b32 s48, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 5
+; SI-NEXT: v_mov_b32_e32 v5, s38
+; SI-NEXT: s_mov_b32 s38, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 3
+; SI-NEXT: s_mov_b32 s68, s17
+; SI-NEXT: v_readlane_b32 s17, v37, 1
+; SI-NEXT: s_mov_b32 s24, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 63
+; SI-NEXT: s_mov_b32 s52, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 61
+; SI-NEXT: v_mov_b32_e32 v23, s80
+; SI-NEXT: s_mov_b32 s80, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 59
+; SI-NEXT: v_mov_b32_e32 v2, s66
+; SI-NEXT: s_mov_b32 s66, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 55
+; SI-NEXT: v_mov_b32_e32 v13, s82
+; SI-NEXT: s_mov_b32 s82, s17
+; SI-NEXT: v_readlane_b32 s17, v36, 53
+; SI-NEXT: v_mov_b32_e32 v1, s70
+; SI-NEXT: v_mov_b32_e32 v8, s93
+; SI-NEXT: v_mov_b32_e32 v22, s99
+; SI-NEXT: v_mov_b32_e32 v16, s81
+; SI-NEXT: v_mov_b32_e32 v17, s87
+; SI-NEXT: v_mov_b32_e32 v20, s83
; SI-NEXT: v_readlane_b32 s89, v35, 0
-; SI-NEXT: v_readlane_b32 s51, v37, 38
; SI-NEXT: v_readlane_b32 s91, v35, 1
-; SI-NEXT: v_readlane_b32 s97, v36, 46
; SI-NEXT: v_readlane_b32 s93, v35, 2
-; SI-NEXT: v_readlane_b32 s85, v36, 44
-; SI-NEXT: v_readlane_b32 s55, v36, 42
-; SI-NEXT: v_readlane_b32 s65, v36, 40
-; SI-NEXT: v_readlane_b32 s87, v36, 36
-; SI-NEXT: v_readlane_b32 s99, v37, 12
-; SI-NEXT: v_readlane_b32 s21, v37, 10
-; SI-NEXT: v_readlane_b32 s49, v37, 8
-; SI-NEXT: v_readlane_b32 s39, v37, 6
-; SI-NEXT: v_readlane_b32 s69, v37, 4
-; SI-NEXT: v_readlane_b32 s25, v37, 2
-; SI-NEXT: v_readlane_b32 s53, v37, 0
-; SI-NEXT: v_readlane_b32 s81, v36, 62
-; SI-NEXT: v_readlane_b32 s67, v36, 60
-; SI-NEXT: v_readlane_b32 s83, v36, 56
-; SI-NEXT: v_readlane_b32 s71, v36, 54
+; SI-NEXT: s_mov_b32 s70, s17
; SI-NEXT: .LBB95_6: ; %end
; SI-NEXT: s_lshl_b32 s16, s16, 8
; SI-NEXT: s_and_b32 s17, s74, 0xff
@@ -182778,13 +182268,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_mov_b32_e32 v31, s16
; SI-NEXT: s_and_b32 s16, s63, 0xff
; SI-NEXT: s_lshl_b32 s17, s89, 8
-; SI-NEXT: v_and_b32_e32 v19, 0xff, v19
+; SI-NEXT: v_and_b32_e32 v18, 0xff, v18
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; SI-NEXT: v_lshlrev_b32_e32 v30, 24, v30
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_or_b32_e32 v19, v30, v19
-; SI-NEXT: v_or_b32_e32 v19, s16, v19
+; SI-NEXT: v_or_b32_e32 v18, v30, v18
+; SI-NEXT: v_or_b32_e32 v18, s16, v18
; SI-NEXT: s_lshl_b32 s16, s90, 8
; SI-NEXT: s_and_b32 s17, s58, 0xff
; SI-NEXT: s_or_b32 s16, s17, s16
@@ -182836,13 +182326,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_and_b32 s16, s61, 0xff
; SI-NEXT: s_lshl_b32 s17, s29, 8
; SI-NEXT: v_and_b32_e32 v11, 0xff, v11
-; SI-NEXT: v_readlane_b32 s46, v36, 47
; SI-NEXT: buffer_store_dword v33, v0, s[0:3], 0 offen
; SI-NEXT: buffer_store_dword v26, v0, s[0:3], 0 offen offset:4
; SI-NEXT: buffer_store_dword v32, v0, s[0:3], 0 offen offset:8
; SI-NEXT: buffer_store_dword v22, v0, s[0:3], 0 offen offset:12
; SI-NEXT: buffer_store_dword v31, v0, s[0:3], 0 offen offset:16
-; SI-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:20
+; SI-NEXT: buffer_store_dword v18, v0, s[0:3], 0 offen offset:20
; SI-NEXT: buffer_store_dword v30, v0, s[0:3], 0 offen offset:24
; SI-NEXT: buffer_store_dword v15, v0, s[0:3], 0 offen offset:28
; SI-NEXT: buffer_store_dword v29, v0, s[0:3], 0 offen offset:32
@@ -182852,19 +182341,18 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_lshlrev_b32_e32 v12, 24, v27
-; SI-NEXT: v_readlane_b32 s47, v36, 48
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: s_lshl_b32 s17, s46, 8
-; SI-NEXT: v_readlane_b32 s46, v36, 49
+; SI-NEXT: v_readlane_b32 s17, v36, 47
; SI-NEXT: v_or_b32_e32 v11, s16, v11
; SI-NEXT: s_and_b32 s16, s56, 0xff
-; SI-NEXT: v_readlane_b32 s47, v36, 50
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s46, 0xff
-; SI-NEXT: v_readlane_b32 s46, v36, 51
+; SI-NEXT: v_readlane_b32 s17, v36, 49
+; SI-NEXT: s_and_b32 s17, s17, 0xff
+; SI-NEXT: v_readlane_b32 s18, v36, 51
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s46, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -182886,9 +182374,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s17, s70, 8
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s82, 0xff
-; SI-NEXT: v_readlane_b32 s28, v36, 57
+; SI-NEXT: v_readlane_b32 s18, v36, 57
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: s_lshl_b32 s18, s28, 24
+; SI-NEXT: s_lshl_b32 s18, s18, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
@@ -182965,7 +182453,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, s14
; SI-NEXT: s_and_b32 s14, s15, 0xff
-; SI-NEXT: s_lshl_b32 s15, s19, 8
+; SI-NEXT: s_lshl_b32 s15, s35, 8
; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
; SI-NEXT: buffer_store_dword v7, v0, s[0:3], 0 offen offset:80
; SI-NEXT: s_or_b32 s14, s14, s15
@@ -182977,15 +182465,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v6, s14, v6
; SI-NEXT: v_readlane_b32 s14, v37, 13
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v37, 14
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v37, 15
-; SI-NEXT: v_readlane_b32 s15, v37, 16
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v37, 17
+; SI-NEXT: v_readlane_b32 s15, v37, 17
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -182993,27 +182479,25 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v6, s12
; SI-NEXT: s_and_b32 s12, s13, 0xff
-; SI-NEXT: s_lshl_b32 s13, s35, 8
+; SI-NEXT: s_lshl_b32 s13, s31, 8
; SI-NEXT: v_and_b32_e32 v5, 0xff, v5
; SI-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:88
; SI-NEXT: s_or_b32 s12, s12, s13
; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v18
+; SI-NEXT: v_lshlrev_b32_e32 v6, 24, v19
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: v_or_b32_e32 v5, s12, v5
; SI-NEXT: v_readlane_b32 s12, v37, 19
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s13, v37, 20
; SI-NEXT: s_lshl_b32 s12, s12, 8
; SI-NEXT: s_or_b32 s10, s10, s12
; SI-NEXT: v_readlane_b32 s12, v37, 21
-; SI-NEXT: v_readlane_b32 s13, v37, 22
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s14, v37, 23
+; SI-NEXT: v_readlane_b32 s13, v37, 23
; SI-NEXT: s_lshl_b32 s12, s12, 16
-; SI-NEXT: s_lshl_b32 s13, s14, 24
+; SI-NEXT: s_lshl_b32 s13, s13, 24
; SI-NEXT: s_and_b32 s10, s10, 0xffff
; SI-NEXT: s_or_b32 s12, s13, s12
; SI-NEXT: s_or_b32 s10, s10, s12
@@ -183021,7 +182505,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, s10
; SI-NEXT: s_and_b32 s10, s11, 0xff
-; SI-NEXT: s_lshl_b32 s11, s31, 8
+; SI-NEXT: s_lshl_b32 s11, s23, 8
; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
; SI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:96
; SI-NEXT: s_or_b32 s10, s10, s11
@@ -183033,15 +182517,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v4, s10, v4
; SI-NEXT: v_readlane_b32 s10, v37, 25
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s11, v37, 26
; SI-NEXT: s_lshl_b32 s10, s10, 8
; SI-NEXT: s_or_b32 s8, s8, s10
; SI-NEXT: v_readlane_b32 s10, v37, 27
-; SI-NEXT: v_readlane_b32 s11, v37, 28
; SI-NEXT: s_and_b32 s10, s10, 0xff
-; SI-NEXT: v_readlane_b32 s12, v37, 29
+; SI-NEXT: v_readlane_b32 s11, v37, 29
; SI-NEXT: s_lshl_b32 s10, s10, 16
-; SI-NEXT: s_lshl_b32 s11, s12, 24
+; SI-NEXT: s_lshl_b32 s11, s11, 24
; SI-NEXT: s_and_b32 s8, s8, 0xffff
; SI-NEXT: s_or_b32 s10, s11, s10
; SI-NEXT: s_or_b32 s8, s8, s10
@@ -183061,15 +182543,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v3, s8, v3
; SI-NEXT: v_readlane_b32 s8, v37, 31
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s9, v37, 32
; SI-NEXT: s_lshl_b32 s8, s8, 8
; SI-NEXT: s_or_b32 s6, s6, s8
; SI-NEXT: v_readlane_b32 s8, v37, 33
-; SI-NEXT: v_readlane_b32 s9, v37, 34
; SI-NEXT: s_and_b32 s8, s8, 0xff
-; SI-NEXT: v_readlane_b32 s10, v37, 35
+; SI-NEXT: v_readlane_b32 s9, v37, 35
; SI-NEXT: s_lshl_b32 s8, s8, 16
-; SI-NEXT: s_lshl_b32 s9, s10, 24
+; SI-NEXT: s_lshl_b32 s9, s9, 24
; SI-NEXT: s_and_b32 s6, s6, 0xffff
; SI-NEXT: s_or_b32 s8, s9, s8
; SI-NEXT: s_or_b32 s6, s6, s8
@@ -183091,11 +182571,10 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_lshl_b32 s6, s50, 8
; SI-NEXT: s_or_b32 s4, s4, s6
; SI-NEXT: v_readlane_b32 s6, v37, 41
-; SI-NEXT: v_readlane_b32 s7, v37, 42
; SI-NEXT: s_and_b32 s6, s6, 0xff
-; SI-NEXT: v_readlane_b32 s8, v37, 39
+; SI-NEXT: v_readlane_b32 s7, v37, 39
; SI-NEXT: s_lshl_b32 s6, s6, 16
-; SI-NEXT: s_lshl_b32 s7, s8, 24
+; SI-NEXT: s_lshl_b32 s7, s7, 24
; SI-NEXT: s_and_b32 s4, s4, 0xffff
; SI-NEXT: s_or_b32 s6, s7, s6
; SI-NEXT: s_or_b32 s4, s4, s6
@@ -183103,7 +182582,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: s_and_b32 s4, s5, 0xff
-; SI-NEXT: s_lshl_b32 s5, s23, 8
+; SI-NEXT: s_lshl_b32 s5, s19, 8
; SI-NEXT: v_and_b32_e32 v1, 0xff, v1
; SI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
; SI-NEXT: s_or_b32 s4, s4, s5
@@ -183114,13 +182593,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: v_or_b32_e32 v1, s4, v1
; SI-NEXT: v_readlane_b32 s30, v34, 34
-; SI-NEXT: v_readlane_b32 s47, v36, 52
-; SI-NEXT: v_readlane_b32 s29, v36, 58
-; SI-NEXT: v_readlane_b32 s17, v37, 18
-; SI-NEXT: v_readlane_b32 s15, v37, 24
-; SI-NEXT: v_readlane_b32 s13, v37, 30
-; SI-NEXT: v_readlane_b32 s11, v37, 36
-; SI-NEXT: v_readlane_b32 s9, v37, 40
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; SI-NEXT: v_readlane_b32 s31, v34, 35
; SI-NEXT: v_readlane_b32 s99, v34, 33
@@ -183563,20 +183035,20 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_lshr_b32 s46, s45, 16
; VI-NEXT: v_mov_b32_e32 v8, 0x200
; VI-NEXT: v_add_f16_e32 v1, s46, v8
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_add_f16_e32 v2, s45, v8
; VI-NEXT: s_lshr_b32 s45, s44, 16
; VI-NEXT: v_or_b32_e32 v38, v2, v1
; VI-NEXT: v_add_f16_e32 v1, s45, v8
-; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_add_f16_e32 v2, s44, v8
; VI-NEXT: s_lshr_b32 s44, s43, 16
; VI-NEXT: v_or_b32_e32 v37, v2, v1
; VI-NEXT: v_add_f16_e32 v1, s44, v8
-; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_add_f16_e32 v2, s43, v8
@@ -183785,8 +183257,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v22
-; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v21
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v18
@@ -183812,40 +183284,49 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_lshrrev_b64 v[1:2], 24, v[23:24]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v33
+; VI-NEXT: v_lshrrev_b64 v[16:17], 24, v[15:16]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v20
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; VI-NEXT: v_lshrrev_b64 v[17:18], 24, v[33:34]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v19
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v26
-; VI-NEXT: v_lshrrev_b64 v[16:17], 24, v[15:16]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; VI-NEXT: v_lshrrev_b64 v[18:19], 24, v[19:20]
; VI-NEXT: v_lshrrev_b32_e32 v3, 8, v24
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; VI-NEXT: v_lshrrev_b64 v[1:2], 24, v[25:26]
; VI-NEXT: v_bfe_u32 v26, v40, 8, 8
; VI-NEXT: v_lshrrev_b64 v[8:9], 24, v[12:13]
-; VI-NEXT: v_lshrrev_b64 v[17:18], 24, v[33:34]
+; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[31:32]
; VI-NEXT: v_lshrrev_b32_e32 v4, 8, v23
; VI-NEXT: v_mov_b32_e32 v23, v3
; VI-NEXT: v_bfe_u32 v3, v43, 8, 8
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v26, v57, 8, 8
-; VI-NEXT: v_lshrrev_b64 v[18:19], 24, v[19:20]
; VI-NEXT: v_lshrrev_b32_e32 v9, 8, v28
; VI-NEXT: v_lshrrev_b32_e32 v13, 8, v27
+; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; VI-NEXT: v_lshrrev_b32_e32 v25, 8, v38
+; VI-NEXT: v_lshrrev_b32_e32 v20, 8, v37
+; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[37:38]
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v58, 8, 8
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v26, v42, 8, 8
; VI-NEXT: v_lshrrev_b32_e32 v33, 8, v31
-; VI-NEXT: v_lshrrev_b64 v[19:20], 24, v[31:32]
; VI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v55
; VI-NEXT: v_lshrrev_b32_e32 v1, 8, v54
; VI-NEXT: v_lshrrev_b64 v[11:12], 24, v[54:55]
+; VI-NEXT: v_mov_b32_e32 v55, v6
+; VI-NEXT: v_bfe_u32 v38, v6, 8, 8
+; VI-NEXT: v_mov_b32_e32 v6, v7
+; VI-NEXT: v_bfe_u32 v24, v7, 8, 8
; VI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v15, v5, 8, 8
; VI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
@@ -183860,27 +183341,19 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
; VI-NEXT: v_bfe_u32 v3, v60, 8, 8
-; VI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; VI-NEXT: v_mov_b32_e32 v54, v50
-; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; VI-NEXT: v_mov_b32_e32 v7, v50
+; VI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; VI-NEXT: v_mov_b32_e32 v39, v13
; VI-NEXT: v_mov_b32_e32 v13, v48
; VI-NEXT: v_mov_b32_e32 v48, v9
; VI-NEXT: v_mov_b32_e32 v9, v41
; VI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; VI-NEXT: v_lshrrev_b32_e32 v29, 8, v25
-; VI-NEXT: v_lshrrev_b32_e32 v25, 8, v38
-; VI-NEXT: v_lshrrev_b32_e32 v20, 8, v37
-; VI-NEXT: v_lshrrev_b64 v[37:38], 24, v[37:38]
; VI-NEXT: v_lshrrev_b32_e32 v34, 8, v32
-; VI-NEXT: v_lshrrev_b32_e32 v30, 8, v55
-; VI-NEXT: v_mov_b32_e32 v55, v6
-; VI-NEXT: v_bfe_u32 v38, v6, 8, 8
-; VI-NEXT: v_mov_b32_e32 v6, v7
-; VI-NEXT: v_bfe_u32 v24, v7, 8, 8
; VI-NEXT: v_bfe_u32 v32, v51, 8, 8
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
; VI-NEXT: s_waitcnt vmcnt(6)
@@ -183889,8 +183362,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_bfe_u32 v26, v50, 8, 8
; VI-NEXT: s_branch .LBB95_6
; VI-NEXT: .LBB95_5:
-; VI-NEXT: v_mov_b32_e32 v1, s45
-; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s42
; VI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v1, s43
@@ -184083,8 +183554,8 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v3, s56
-; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
+; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
; VI-NEXT: v_mov_b32_e32 v3, s78
; VI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
@@ -184118,8 +183589,9 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_readlane_b32 s4, v62, 57
; VI-NEXT: v_mov_b32_e32 v3, s34
; VI-NEXT: v_mov_b32_e32 v35, s44
+; VI-NEXT: v_mov_b32_e32 v54, s45
; VI-NEXT: v_mov_b32_e32 v59, s16
-; VI-NEXT: v_mov_b32_e32 v54, s17
+; VI-NEXT: v_mov_b32_e32 v7, s17
; VI-NEXT: v_mov_b32_e32 v61, s18
; VI-NEXT: v_mov_b32_e32 v52, s19
; VI-NEXT: v_mov_b32_e32 v14, s71
@@ -184210,20 +183682,18 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v10, v3, v10, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v9, v3, v7, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v9, v3, v8, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen
-; VI-NEXT: v_perm_b32 v19, v54, v34, s4
+; VI-NEXT: v_perm_b32 v19, v7, v34, s4
; VI-NEXT: v_or_b32_e32 v19, v19, v27
; VI-NEXT: buffer_store_dword v19, v0, s[0:3], 0 offen offset:4
; VI-NEXT: s_waitcnt vmcnt(2)
-; VI-NEXT: v_perm_b32 v8, v3, v7, s4
+; VI-NEXT: v_perm_b32 v8, v3, v8, s4
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v19, v61, v3, s4
@@ -184326,12 +183796,10 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_perm_b32 v2, v35, v20, s4
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v14
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v3, v60, v12, s4
+; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:64
+; VI-NEXT: v_perm_b32 v2, v54, v25, s4
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v2, v2, v25, s4
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
@@ -184347,12 +183815,12 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_perm_b32 v1, v1, v30, s4
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v2, v2, v5, s4
+; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v29, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -184368,14 +183836,13 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
-; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_perm_b32 v1, v1, v4, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v1, v1, v4, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
@@ -184390,10 +183857,10 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v2, v41, v2, s4
; VI-NEXT: v_perm_b32 v1, v1, v39, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v2, v41, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
@@ -184411,8 +183878,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v44, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -184432,8 +183898,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v47, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -184453,8 +183918,7 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v45, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -185378,27 +184842,22 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-NEXT: .LBB95_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
-; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v33, v33, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v21, v35, v21, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v23, v34, v23, s4
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v16, v36, v16, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v24, v38, v24, s4
; GFX9-NEXT: v_perm_b32 v17, v28, v17, s4
-; GFX9-NEXT: v_perm_b32 v48, v48, v55, s4
-; GFX9-NEXT: v_perm_b32 v49, v49, v50, s4
-; GFX9-NEXT: v_perm_b32 v50, v54, v43, s4
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v21, v35, v21, s4
; GFX9-NEXT: v_perm_b32 v39, v39, v44, s4
+; GFX9-NEXT: v_perm_b32 v48, v48, v55, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; GFX9-NEXT: v_or_b32_e32 v39, v48, v39
-; GFX9-NEXT: v_perm_b32 v23, v34, v23, s4
+; GFX9-NEXT: v_perm_b32 v49, v49, v50, s4
+; GFX9-NEXT: v_perm_b32 v50, v54, v43, s4
+; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
+; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
; GFX9-NEXT: v_perm_b32 v32, v32, v59, s4
; GFX9-NEXT: v_perm_b32 v29, v29, v57, s4
; GFX9-NEXT: v_perm_b32 v30, v30, v56, s4
@@ -185440,45 +184899,43 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v36, v58, v52, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v22, v25, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v35, v61, v35, s4
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v18, v18, v54, s4
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v34, v60, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v58, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v18, v18, v22, s4
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX9-NEXT: v_or_b32_e32 v18, v21, v18
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v38, v47, v52, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v25, v26, v15, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v34, v60, v54, s4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v28, v51, v52, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v47, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v35, v61, v22, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v28, v51, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v22, v25, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v25, v26, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v26, v15, v51, s4
+; GFX9-NEXT: v_perm_b32 v26, v26, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v19, v19, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v20, v20, v15, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
+; GFX9-NEXT: v_perm_b32 v15, v51, v15, s4
; GFX9-NEXT: buffer_store_dword v39, v0, s[0:3], 0 offen
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
@@ -185594,7 +185051,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v11, v13, s4
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v13, v14, v13, s4
@@ -185615,7 +185071,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v9, v11, s4
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v12, v11, s4
@@ -185636,7 +185091,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v7, v9, s4
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v10, v9, s4
@@ -185657,7 +185111,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v5, v7, s4
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v8, v7, s4
@@ -185678,7 +185131,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v3, v5, s4
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v6, v5, s4
@@ -185697,7 +185149,6 @@ define inreg <128 x i8> @bitcast_v64f16_to_v128i8_scalar(<64 x half> inreg %a, i
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:116
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4
@@ -200193,50 +199644,42 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: .LBB98_4: ; %end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; VI-NEXT: s_mov_b32 s4, 0xc0c0004
-; VI-NEXT: s_waitcnt vmcnt(14)
+; VI-NEXT: v_perm_b32 v10, v49, v31, s4
+; VI-NEXT: s_waitcnt vmcnt(10)
; VI-NEXT: v_perm_b32 v1, v46, v1, s4
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; VI-NEXT: s_waitcnt vmcnt(12)
+; VI-NEXT: s_waitcnt vmcnt(8)
; VI-NEXT: v_perm_b32 v2, v41, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; VI-NEXT: s_waitcnt vmcnt(10)
+; VI-NEXT: s_waitcnt vmcnt(7)
; VI-NEXT: v_perm_b32 v3, v47, v3, s4
-; VI-NEXT: s_waitcnt vmcnt(8)
+; VI-NEXT: s_waitcnt vmcnt(6)
; VI-NEXT: v_perm_b32 v4, v39, v4, s4
-; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v6, v6, v7, s4
-; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; VI-NEXT: v_perm_b32 v9, v50, v9, s4
+; VI-NEXT: s_waitcnt vmcnt(2)
; VI-NEXT: v_perm_b32 v11, v11, v12, s4
-; VI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; VI-NEXT: v_or_b32_e32 v1, v11, v1
-; VI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v6, v6, v7, s4
+; VI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v5, v40, v5, s4
+; VI-NEXT: v_perm_b32 v8, v44, v8, s4
; VI-NEXT: s_waitcnt vmcnt(3)
-; VI-NEXT: v_perm_b32 v10, v49, v31, s4
; VI-NEXT: v_perm_b32 v7, v42, v7, s4
-; VI-NEXT: v_perm_b32 v8, v44, v8, s4
-; VI-NEXT: v_perm_b32 v9, v50, v9, s4
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v11, s4
; VI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
@@ -200397,8 +199840,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -200418,8 +199860,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v63, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -200440,8 +199881,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v62, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -200461,8 +199901,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v48, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -200483,8 +199922,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v52, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -200505,8 +199943,7 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; VI-NEXT: v_perm_b32 v1, v1, v2, s4
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v2, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -201279,9 +200716,8 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:76
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v1, v21, v47, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v46, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:80
@@ -201295,14 +200731,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:84
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v23, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:88
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
@@ -201315,14 +200751,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:92
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v25, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:96
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -201335,14 +200771,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:100
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v27, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:104
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
@@ -201355,14 +200791,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:108
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v29, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:112
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
@@ -201375,14 +200811,14 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) #0 {
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:116
+; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; GFX9-NEXT: s_nop 0
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4
-; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v31, v1, s4
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:120
; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
@@ -203495,14 +202931,12 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_lshr_b32 s67, s5, 8
; SI-NEXT: v_writelane_b32 v22, s16, 53
; SI-NEXT: .LBB99_5: ; %end
-; SI-NEXT: v_readlane_b32 s18, v21, 23
-; SI-NEXT: v_readlane_b32 s19, v21, 24
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 25
+; SI-NEXT: v_readlane_b32 s17, v21, 23
; SI-NEXT: s_and_b32 s16, s64, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 26
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 25
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 27
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203517,23 +202951,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 4
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 31
-; SI-NEXT: v_readlane_b32 s19, v21, 28
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 29
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 30
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 31
+; SI-NEXT: v_readlane_b32 s17, v21, 29
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s92, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 32
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 31
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 33
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203550,23 +202981,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 3
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 33
-; SI-NEXT: v_readlane_b32 s19, v21, 34
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 35
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 36
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 37
+; SI-NEXT: v_readlane_b32 s17, v21, 35
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:8
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s62, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 38
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 37
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 39
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203583,23 +203011,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 2
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 35
-; SI-NEXT: v_readlane_b32 s19, v21, 40
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 41
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 42
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 43
+; SI-NEXT: v_readlane_b32 s17, v21, 41
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s58, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 44
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 43
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 45
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203616,23 +203041,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 1
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 37
-; SI-NEXT: v_readlane_b32 s19, v21, 46
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 47
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 48
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 49
+; SI-NEXT: v_readlane_b32 s17, v21, 47
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:24
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s46, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 50
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 49
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 51
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203649,23 +203071,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_readlane_b32 s17, v21, 0
; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 39
-; SI-NEXT: v_readlane_b32 s19, v21, 52
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 53
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 54
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 55
+; SI-NEXT: v_readlane_b32 s17, v21, 53
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s60, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 56
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 55
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 57
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203681,23 +203100,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s35, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 41
-; SI-NEXT: v_readlane_b32 s19, v21, 58
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v21, 59
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v21, 60
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v21, 61
+; SI-NEXT: v_readlane_b32 s17, v21, 59
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:40
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s56, 0xff
-; SI-NEXT: v_readlane_b32 s19, v21, 62
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v21, 61
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v21, 63
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203713,23 +203129,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s99, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 43
-; SI-NEXT: v_readlane_b32 s19, v22, 0
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 1
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 2
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 3
+; SI-NEXT: v_readlane_b32 s17, v22, 1
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s44, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 4
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 3
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 5
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203745,23 +203158,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s71, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 45
-; SI-NEXT: v_readlane_b32 s19, v22, 6
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 7
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 8
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 9
+; SI-NEXT: v_readlane_b32 s17, v22, 7
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 10
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 9
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 11
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203777,23 +203187,20 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_or_b32 s16, s16, s17
; SI-NEXT: s_and_b32 s17, s55, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 47
-; SI-NEXT: v_readlane_b32 s19, v22, 12
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
-; SI-NEXT: s_or_b32 s17, s18, s17
-; SI-NEXT: v_readlane_b32 s18, v22, 13
; SI-NEXT: s_and_b32 s16, s16, 0xffff
-; SI-NEXT: v_readlane_b32 s19, v22, 14
+; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v22, 15
+; SI-NEXT: v_readlane_b32 s17, v22, 13
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:64
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: s_and_b32 s16, s40, 0xff
-; SI-NEXT: v_readlane_b32 s19, v22, 16
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v22, 15
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: v_readlane_b32 s18, v22, 17
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s18, 24
@@ -203819,16 +203226,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_readlane_b32 s16, v22, 19
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s17, v22, 20
; SI-NEXT: s_lshl_b32 s16, s16, 8
-; SI-NEXT: v_readlane_b32 s19, v22, 18
; SI-NEXT: s_or_b32 s14, s14, s16
; SI-NEXT: v_readlane_b32 s16, v22, 21
-; SI-NEXT: v_readlane_b32 s17, v22, 22
; SI-NEXT: s_and_b32 s16, s16, 0xff
-; SI-NEXT: v_readlane_b32 s18, v22, 23
+; SI-NEXT: v_readlane_b32 s17, v22, 23
; SI-NEXT: s_lshl_b32 s16, s16, 16
-; SI-NEXT: s_lshl_b32 s17, s18, 24
+; SI-NEXT: s_lshl_b32 s17, s17, 24
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_or_b32 s16, s17, s16
; SI-NEXT: s_or_b32 s14, s14, s16
@@ -203851,15 +203255,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: v_mov_b32_e32 v1, s14
; SI-NEXT: v_readlane_b32 s14, v22, 25
; SI-NEXT: s_and_b32 s12, s12, 0xff
-; SI-NEXT: v_readlane_b32 s15, v22, 26
; SI-NEXT: s_lshl_b32 s14, s14, 8
; SI-NEXT: s_or_b32 s12, s12, s14
; SI-NEXT: v_readlane_b32 s14, v22, 27
-; SI-NEXT: v_readlane_b32 s15, v22, 28
; SI-NEXT: s_and_b32 s14, s14, 0xff
-; SI-NEXT: v_readlane_b32 s16, v22, 29
+; SI-NEXT: v_readlane_b32 s15, v22, 29
; SI-NEXT: s_lshl_b32 s14, s14, 16
-; SI-NEXT: s_lshl_b32 s15, s16, 24
+; SI-NEXT: s_lshl_b32 s15, s15, 24
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_or_b32 s14, s15, s14
; SI-NEXT: s_or_b32 s12, s12, s14
@@ -203976,8 +203378,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v20, 34
-; SI-NEXT: v_readlane_b32 s19, v22, 24
-; SI-NEXT: v_readlane_b32 s17, v22, 30
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; SI-NEXT: v_readlane_b32 s31, v20, 35
; SI-NEXT: v_readlane_b32 s99, v20, 33
@@ -204783,19 +204183,18 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v26, s84, v26, v1
; VI-NEXT: v_or_b32_e32 v2, v8, v2
; VI-NEXT: v_perm_b32 v7, s21, v10, v1
-; VI-NEXT: v_mov_b32_e32 v15, s16
+; VI-NEXT: v_mov_b32_e32 v16, s16
; VI-NEXT: v_readlane_b32 s16, v33, 32
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:16
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v26
; VI-NEXT: v_mov_b32_e32 v10, s16
-; VI-NEXT: v_perm_b32 v15, s81, v15, v1
+; VI-NEXT: v_perm_b32 v16, s81, v16, v1
; VI-NEXT: v_or_b32_e32 v2, v7, v2
-; VI-NEXT: v_readlane_b32 s17, v33, 1
; VI-NEXT: v_readlane_b32 s16, v33, 31
; VI-NEXT: v_perm_b32 v11, s22, v10, v1
; VI-NEXT: v_mov_b32_e32 v27, s64
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:20
-; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v15
+; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v16
; VI-NEXT: v_mov_b32_e32 v13, s16
; VI-NEXT: v_readlane_b32 s16, v33, 2
; VI-NEXT: v_perm_b32 v27, s70, v27, v1
@@ -204808,25 +204207,23 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_mov_b32_e32 v13, s16
; VI-NEXT: v_perm_b32 v20, s54, v20, v1
; VI-NEXT: v_or_b32_e32 v2, v10, v2
-; VI-NEXT: v_readlane_b32 s17, v33, 3
; VI-NEXT: v_readlane_b32 s16, v33, 29
; VI-NEXT: v_perm_b32 v14, s24, v13, v1
; VI-NEXT: v_mov_b32_e32 v28, s50
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:28
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v20
-; VI-NEXT: v_mov_b32_e32 v16, s16
+; VI-NEXT: v_mov_b32_e32 v15, s16
; VI-NEXT: v_readlane_b32 s16, v33, 4
; VI-NEXT: v_perm_b32 v28, s67, v28, v1
; VI-NEXT: v_or_b32_e32 v2, v14, v2
-; VI-NEXT: v_perm_b32 v13, s25, v16, v1
+; VI-NEXT: v_perm_b32 v13, s25, v15, v1
; VI-NEXT: v_mov_b32_e32 v21, s16
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:32
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v28
-; VI-NEXT: v_mov_b32_e32 v16, vcc_hi
+; VI-NEXT: v_mov_b32_e32 v15, vcc_hi
; VI-NEXT: v_perm_b32 v21, s85, v21, v1
; VI-NEXT: v_or_b32_e32 v2, v13, v2
-; VI-NEXT: v_readlane_b32 s17, v33, 5
-; VI-NEXT: v_perm_b32 v17, s26, v16, v1
+; VI-NEXT: v_perm_b32 v17, s26, v15, v1
; VI-NEXT: v_mov_b32_e32 v29, s65
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:36
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v21
@@ -204834,14 +204231,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_readlane_b32 s16, v33, 6
; VI-NEXT: v_perm_b32 v29, s51, v29, v1
; VI-NEXT: v_or_b32_e32 v2, v17, v2
-; VI-NEXT: v_perm_b32 v16, s27, v18, v1
+; VI-NEXT: v_perm_b32 v15, s27, v18, v1
; VI-NEXT: v_mov_b32_e32 v22, s16
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:40
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v29
; VI-NEXT: v_mov_b32_e32 v18, s79
; VI-NEXT: v_perm_b32 v22, s86, v22, v1
-; VI-NEXT: v_or_b32_e32 v2, v16, v2
-; VI-NEXT: v_readlane_b32 s17, v33, 7
+; VI-NEXT: v_or_b32_e32 v2, v15, v2
; VI-NEXT: v_perm_b32 v19, s28, v18, v1
; VI-NEXT: v_mov_b32_e32 v30, s52
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:44
@@ -204867,7 +204263,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s71, v3, v1
; VI-NEXT: v_perm_b32 v2, s45, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s17, v33, 9
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_readlane_b32 s16, v33, 60
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:60
@@ -204888,7 +204283,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s43, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s17, v33, 11
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_readlane_b32 s16, v33, 55
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:68
@@ -204909,14 +204303,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s16, v3, v1
; VI-NEXT: v_perm_b32 v2, s41, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s17, v33, 13
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_readlane_b32 s16, v33, 50
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:76
; VI-NEXT: v_mov_b32_e32 v2, s16
-; VI-NEXT: v_readlane_b32 s16, v33, 14
; VI-NEXT: v_perm_b32 v2, s14, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s16
+; VI-NEXT: v_readlane_b32 s14, v33, 14
+; VI-NEXT: v_mov_b32_e32 v3, s14
; VI-NEXT: v_readlane_b32 s14, v33, 49
; VI-NEXT: v_perm_b32 v3, s14, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
@@ -204934,9 +204327,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_readlane_b32 s14, v33, 45
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:84
; VI-NEXT: v_mov_b32_e32 v2, s14
-; VI-NEXT: v_readlane_b32 s14, v33, 16
; VI-NEXT: v_perm_b32 v2, s12, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s14
+; VI-NEXT: v_readlane_b32 s12, v33, 16
+; VI-NEXT: v_mov_b32_e32 v3, s12
; VI-NEXT: v_readlane_b32 s12, v33, 44
; VI-NEXT: v_perm_b32 v3, s12, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
@@ -204954,9 +204347,9 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_readlane_b32 s12, v33, 40
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:92
; VI-NEXT: v_mov_b32_e32 v2, s12
-; VI-NEXT: v_readlane_b32 s12, v33, 18
; VI-NEXT: v_perm_b32 v2, s10, v2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s12
+; VI-NEXT: v_readlane_b32 s10, v33, 18
+; VI-NEXT: v_mov_b32_e32 v3, s10
; VI-NEXT: v_readlane_b32 s10, v33, 39
; VI-NEXT: v_perm_b32 v3, s10, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
@@ -204967,13 +204360,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s90, v3, v1
; VI-NEXT: v_perm_b32 v2, s11, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s10, v33, 20
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s10
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:100
; VI-NEXT: v_mov_b32_e32 v2, s88
-; VI-NEXT: v_perm_b32 v3, s69, v3, v1
; VI-NEXT: v_perm_b32 v2, s8, v2, v1
+; VI-NEXT: v_readlane_b32 s8, v33, 20
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: v_perm_b32 v3, s69, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s76
@@ -204982,13 +204375,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s68, v3, v1
; VI-NEXT: v_perm_b32 v2, s9, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s8, v33, 22
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s8
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:108
; VI-NEXT: v_mov_b32_e32 v2, s63
-; VI-NEXT: v_perm_b32 v3, s61, v3, v1
; VI-NEXT: v_perm_b32 v2, s6, v2, v1
+; VI-NEXT: v_readlane_b32 s6, v33, 22
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_perm_b32 v3, s61, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: v_mov_b32_e32 v3, s56
@@ -204997,13 +204390,13 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_perm_b32 v3, s57, v3, v1
; VI-NEXT: v_perm_b32 v2, s7, v2, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-NEXT: v_readlane_b32 s6, v33, 24
; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:116
; VI-NEXT: v_mov_b32_e32 v2, s55
-; VI-NEXT: v_perm_b32 v3, s47, v3, v1
; VI-NEXT: v_perm_b32 v2, s4, v2, v1
+; VI-NEXT: v_readlane_b32 s4, v33, 24
+; VI-NEXT: v_mov_b32_e32 v3, s4
+; VI-NEXT: v_perm_b32 v3, s47, v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v2, v2, v3
; VI-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:120
@@ -205014,12 +204407,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-NEXT: v_or_b32_e32 v1, v2, v1
; VI-NEXT: v_readlane_b32 s30, v32, 30
-; VI-NEXT: v_readlane_b32 s17, v33, 15
-; VI-NEXT: v_readlane_b32 s15, v33, 17
-; VI-NEXT: v_readlane_b32 s13, v33, 19
-; VI-NEXT: v_readlane_b32 s11, v33, 21
-; VI-NEXT: v_readlane_b32 s9, v33, 23
-; VI-NEXT: v_readlane_b32 s7, v33, 25
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:124
; VI-NEXT: v_readlane_b32 s31, v32, 31
; VI-NEXT: v_readlane_b32 s87, v32, 29
@@ -205946,27 +205333,22 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-NEXT: .LBB99_6: ; %end
; GFX9-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
-; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v33, v33, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v21, v35, v21, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v23, v34, v23, s4
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v16, v36, v16, s4
-; GFX9-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v24, v38, v24, s4
; GFX9-NEXT: v_perm_b32 v17, v28, v17, s4
-; GFX9-NEXT: v_perm_b32 v48, v48, v55, s4
-; GFX9-NEXT: v_perm_b32 v49, v49, v50, s4
-; GFX9-NEXT: v_perm_b32 v50, v54, v43, s4
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v21, v35, v21, s4
; GFX9-NEXT: v_perm_b32 v39, v39, v44, s4
+; GFX9-NEXT: v_perm_b32 v48, v48, v55, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v39, 16, v39
; GFX9-NEXT: v_or_b32_e32 v39, v48, v39
-; GFX9-NEXT: v_perm_b32 v23, v34, v23, s4
+; GFX9-NEXT: v_perm_b32 v49, v49, v50, s4
+; GFX9-NEXT: v_perm_b32 v50, v54, v43, s4
+; GFX9-NEXT: v_perm_b32 v37, v37, v52, s4
+; GFX9-NEXT: v_perm_b32 v31, v31, v53, s4
; GFX9-NEXT: v_perm_b32 v32, v32, v59, s4
; GFX9-NEXT: v_perm_b32 v29, v29, v57, s4
; GFX9-NEXT: v_perm_b32 v30, v30, v56, s4
@@ -206008,45 +205390,43 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_perm_b32 v36, v58, v52, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v22, v25, v15, s4
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; GFX9-NEXT: v_perm_b32 v35, v61, v35, s4
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v18, v18, v54, s4
-; GFX9-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v34, v60, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v36, v58, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v18, v18, v22, s4
+; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX9-NEXT: v_or_b32_e32 v18, v21, v18
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v38, v47, v52, s4
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_perm_b32 v25, v26, v15, s4
-; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_perm_b32 v34, v60, v54, s4
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v28, v51, v52, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: v_perm_b32 v38, v47, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_perm_b32 v35, v61, v22, s4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v28, v51, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v26, v15, v51, s4
+; GFX9-NEXT: v_perm_b32 v22, v25, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v25, v26, v15, s4
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_perm_b32 v26, v26, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v19, v19, v15, s4
; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v20, v20, v15, s4
-; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; GFX9-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v15, v15, v51, s4
+; GFX9-NEXT: v_perm_b32 v15, v51, v15, s4
; GFX9-NEXT: buffer_store_dword v39, v0, s[0:3], 0 offen
; GFX9-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
@@ -206162,7 +205542,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v11, v13, s4
; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v13, v14, v13, s4
@@ -206183,7 +205562,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v9, v11, s4
; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v11, v12, v11, s4
@@ -206204,7 +205582,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v7, v9, s4
; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v9, v10, v9, s4
@@ -206225,7 +205602,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v5, v7, s4
; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v7, v8, v7, s4
@@ -206246,7 +205622,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v3, v5, s4
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v6, v5, s4
@@ -206265,7 +205640,6 @@ define inreg <128 x i8> @bitcast_v64i16_to_v128i8_scalar(<64 x i16> inreg %a, i3
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:116
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4
@@ -210641,7 +210015,7 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:696 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
@@ -210671,26 +210045,31 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s6
; SI-NEXT: v_readfirstlane_b32 s76, v8
; SI-NEXT: s_and_b32 s79, s78, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s93
-; SI-NEXT: v_readfirstlane_b32 s46, v15
+; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: s_and_b32 s77, s76, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s79
+; SI-NEXT: v_readfirstlane_b32 s46, v15
+; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
; SI-NEXT: v_readfirstlane_b32 s44, v16
; SI-NEXT: s_and_b32 s47, s46, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s77
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s63
; SI-NEXT: v_readfirstlane_b32 s42, v17
; SI-NEXT: s_and_b32 s45, s44, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s47
; SI-NEXT: s_and_b32 s43, s42, 0xffff0000
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s45
; SI-NEXT: s_and_b32 s14, s24, 0xffff0000
@@ -210699,23 +210078,37 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_lshl_b32 s23, s23, 16
; SI-NEXT: s_and_b32 s40, s17, 0xffff0000
; SI-NEXT: s_lshl_b32 s17, s17, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s43
; SI-NEXT: v_readfirstlane_b32 s30, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; SI-NEXT: v_mul_f32_e64 v1, 1.0, s17
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s23
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
+; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
+; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s15
-; SI-NEXT: s_lshl_b32 s11, s26, 16
-; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
-; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
-; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s18
+; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
+; SI-NEXT: s_lshl_b32 s9, s27, 16
+; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
+; SI-NEXT: s_lshl_b32 s20, s20, 16
+; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
+; SI-NEXT: s_lshl_b32 s13, s25, 16
+; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
+; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
+; SI-NEXT: s_lshl_b32 s34, s34, 16
; SI-NEXT: s_lshl_b32 s92, s92, 16
+; SI-NEXT: v_readfirstlane_b32 s88, v6
+; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
+; SI-NEXT: s_lshl_b32 s88, s88, 16
+; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: v_readfirstlane_b32 s72, v10
; SI-NEXT: s_and_b32 s73, s72, 0xffff0000
; SI-NEXT: s_lshl_b32 s72, s72, 16
@@ -210723,85 +210116,69 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_and_b32 s61, s60, 0xffff0000
; SI-NEXT: s_lshl_b32 s60, s60, 16
; SI-NEXT: v_readfirstlane_b32 s56, v14
-; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
-; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: v_readfirstlane_b32 s58, v13
-; SI-NEXT: v_readfirstlane_b32 s62, v11
; SI-NEXT: v_readfirstlane_b32 s74, v9
-; SI-NEXT: v_readfirstlane_b32 s88, v6
; SI-NEXT: v_readfirstlane_b32 s90, v5
; SI-NEXT: v_readfirstlane_b32 s94, v3
-; SI-NEXT: s_and_b32 s4, s29, 0xffff0000
-; SI-NEXT: s_lshl_b32 s5, s29, 16
+; SI-NEXT: v_readfirstlane_b32 vcc_lo, v2
; SI-NEXT: s_lshl_b32 s7, s28, 16
-; SI-NEXT: s_and_b32 s8, s27, 0xffff0000
-; SI-NEXT: s_lshl_b32 s9, s27, 16
; SI-NEXT: s_and_b32 s10, s26, 0xffff0000
-; SI-NEXT: s_and_b32 s12, s25, 0xffff0000
-; SI-NEXT: s_lshl_b32 s13, s25, 16
-; SI-NEXT: s_and_b32 s25, s22, 0xffff0000
-; SI-NEXT: s_lshl_b32 s22, s22, 16
+; SI-NEXT: s_lshl_b32 s11, s26, 16
; SI-NEXT: s_and_b32 s26, s21, 0xffff0000
; SI-NEXT: s_lshl_b32 s21, s21, 16
-; SI-NEXT: s_and_b32 s27, s20, 0xffff0000
-; SI-NEXT: s_lshl_b32 s20, s20, 16
; SI-NEXT: s_and_b32 s28, s19, 0xffff0000
; SI-NEXT: s_lshl_b32 s19, s19, 16
-; SI-NEXT: s_and_b32 s29, s18, 0xffff0000
-; SI-NEXT: s_lshl_b32 s18, s18, 16
; SI-NEXT: s_and_b32 s41, s16, 0xffff0000
; SI-NEXT: s_lshl_b32 s16, s16, 16
; SI-NEXT: s_lshl_b32 s42, s42, 16
; SI-NEXT: s_lshl_b32 s44, s44, 16
; SI-NEXT: s_lshl_b32 s46, s46, 16
+; SI-NEXT: s_and_b32 s57, s56, 0xffff0000
+; SI-NEXT: s_lshl_b32 s56, s56, 16
; SI-NEXT: s_and_b32 s59, s58, 0xffff0000
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s20
; SI-NEXT: s_lshl_b32 s58, s58, 16
-; SI-NEXT: s_and_b32 s63, s62, 0xffff0000
; SI-NEXT: s_lshl_b32 s62, s62, 16
; SI-NEXT: s_and_b32 s75, s74, 0xffff0000
; SI-NEXT: s_lshl_b32 s74, s74, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:492 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s11
-; SI-NEXT: s_lshl_b32 s76, s76, 16
; SI-NEXT: s_lshl_b32 s78, s78, 16
-; SI-NEXT: s_and_b32 s89, s88, 0xffff0000
-; SI-NEXT: s_lshl_b32 s88, s88, 16
; SI-NEXT: s_and_b32 s91, s90, 0xffff0000
; SI-NEXT: s_lshl_b32 s90, s90, 16
; SI-NEXT: s_and_b32 s95, s94, 0xffff0000
; SI-NEXT: s_lshl_b32 s94, s94, 16
+; SI-NEXT: s_and_b32 vcc_hi, vcc_lo, 0xffff0000
+; SI-NEXT: s_lshl_b32 vcc_lo, vcc_lo, 16
; SI-NEXT: s_and_b32 s31, s30, 0xffff0000
; SI-NEXT: s_lshl_b32 s30, s30, 16
-; SI-NEXT: s_and_b32 s35, s34, 0xffff0000
-; SI-NEXT: s_lshl_b32 s34, s34, 16
; SI-NEXT: v_readfirstlane_b32 s36, v18
; SI-NEXT: s_cmp_lg_u32 s36, 0
-; SI-NEXT: v_mul_f32_e64 v39, 1.0, s41
-; SI-NEXT: v_mul_f32_e64 v44, 1.0, s40
+; SI-NEXT: v_mul_f32_e64 v52, 1.0, s41
+; SI-NEXT: v_mul_f32_e64 v62, 1.0, s40
; SI-NEXT: v_mul_f32_e64 v47, 1.0, s29
; SI-NEXT: v_mul_f32_e64 v3, 1.0, s28
; SI-NEXT: v_mul_f32_e64 v53, 1.0, s27
; SI-NEXT: v_mul_f32_e64 v5, 1.0, s26
; SI-NEXT: v_mul_f32_e64 v9, 1.0, s25
; SI-NEXT: v_mul_f32_e64 v10, 1.0, s24
-; SI-NEXT: v_mul_f32_e64 v62, 1.0, s14
+; SI-NEXT: v_mul_f32_e64 v24, 1.0, s14
; SI-NEXT: v_mul_f32_e64 v14, 1.0, s12
; SI-NEXT: v_mul_f32_e64 v15, 1.0, s10
; SI-NEXT: v_mul_f32_e64 v17, 1.0, s8
; SI-NEXT: v_mul_f32_e64 v18, 1.0, s4
-; SI-NEXT: v_mul_f32_e64 v51, 1.0, s35
+; SI-NEXT: v_mul_f32_e64 v12, 1.0, s35
; SI-NEXT: v_mul_f32_e64 v21, 1.0, s31
-; SI-NEXT: v_mul_f32_e64 v38, 1.0, vcc_hi
+; SI-NEXT: v_mul_f32_e64 v37, 1.0, vcc_hi
; SI-NEXT: v_mul_f32_e64 v29, 1.0, s95
-; SI-NEXT: v_mul_f32_e64 v32, 1.0, s91
+; SI-NEXT: v_mul_f32_e64 v38, 1.0, s91
; SI-NEXT: v_mul_f32_e64 v26, 1.0, s89
; SI-NEXT: v_mul_f32_e64 v42, 1.0, s75
; SI-NEXT: v_mul_f32_e64 v55, 1.0, s73
-; SI-NEXT: v_mul_f32_e64 v56, 1.0, s63
-; SI-NEXT: v_mul_f32_e64 v2, 1.0, s61
-; SI-NEXT: v_mul_f32_e64 v37, 1.0, s59
+; SI-NEXT: v_mul_f32_e64 v56, 1.0, s61
+; SI-NEXT: v_mul_f32_e64 v45, 1.0, s59
+; SI-NEXT: v_mul_f32_e64 v46, 1.0, s57
; SI-NEXT: v_mul_f32_e64 v7, 1.0, s19
; SI-NEXT: v_mul_f32_e64 v4, 1.0, s21
; SI-NEXT: v_mul_f32_e64 v13, 1.0, s13
@@ -210809,866 +210186,825 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: v_mul_f32_e64 v19, 1.0, s5
; SI-NEXT: v_mul_f32_e64 v22, 1.0, s30
; SI-NEXT: v_mul_f32_e64 v35, 1.0, s94
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s22
; SI-NEXT: v_mul_f32_e64 v34, 1.0, s90
; SI-NEXT: v_mul_f32_e64 v31, 1.0, s78
; SI-NEXT: v_mul_f32_e64 v28, 1.0, s74
; SI-NEXT: v_mul_f32_e64 v25, 1.0, s62
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, vcc_lo
; SI-NEXT: v_mul_f32_e64 v27, 1.0, s58
; SI-NEXT: v_mul_f32_e64 v30, 1.0, s46
; SI-NEXT: v_mul_f32_e64 v33, 1.0, s42
-; SI-NEXT: v_mul_f32_e64 v52, 1.0, s16
-; SI-NEXT: v_mul_f32_e64 v50, 1.0, s18
-; SI-NEXT: v_mul_f32_e64 v41, 1.0, s20
-; SI-NEXT: v_mul_f32_e64 v40, 1.0, s22
-; SI-NEXT: v_mul_f32_e64 v45, 1.0, s7
-; SI-NEXT: v_mul_f32_e64 v23, 1.0, s34
-; SI-NEXT: v_mul_f32_e64 v6, 1.0, s88
-; SI-NEXT: v_mul_f32_e64 v61, 1.0, s76
-; SI-NEXT: v_mul_f32_e64 v60, 1.0, s57
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
+; SI-NEXT: v_mul_f32_e64 v61, 1.0, s16
+; SI-NEXT: v_mul_f32_e64 v44, 1.0, s15
+; SI-NEXT: v_mul_f32_e64 v20, 1.0, s11
+; SI-NEXT: v_mul_f32_e64 v8, 1.0, s7
+; SI-NEXT: v_mul_f32_e64 v51, 1.0, vcc_lo
+; SI-NEXT: v_mul_f32_e64 v60, 1.0, s56
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s34
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s92
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s88
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mul_f32_e64 v0, 1.0, s76
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s72
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s60
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mul_f32_e64 v0, 1.0, s56
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: v_mul_f32_e64 v0, 1.0, s44
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:688 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:692 ; 4-byte Folded Spill
; SI-NEXT: s_cbranch_scc0 .LBB101_2
; SI-NEXT: ; %bb.1: ; %cmp.false
-; SI-NEXT: v_mov_b32_e32 v43, v44
-; SI-NEXT: v_mov_b32_e32 v60, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v43
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v39
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v18
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v43, v8
+; SI-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v17
-; SI-NEXT: s_waitcnt expcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v32, v62
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v32
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
; SI-NEXT: v_lshrrev_b32_e32 v36, 16, v29
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v15
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v42
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v10
-; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v53
-; SI-NEXT: v_mov_b32_e32 v24, v6
-; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v38
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v54, 0xffff0000, v56
+; SI-NEXT: v_and_b32_e32 v43, 0xffff0000, v26
+; SI-NEXT: v_mov_b32_e32 v62, v15
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v47
+; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v53
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v18
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[7:8], 16
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v47
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[1:2], v[4:5], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v53
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
-; SI-NEXT: v_lshr_b64 v[3:4], v[13:14], 16
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v12
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v62
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[3:4], v[19:20], 16
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v1, v52
-; SI-NEXT: v_lshr_b64 v[13:14], v[16:17], 16
-; SI-NEXT: v_mov_b32_e32 v52, v15
-; SI-NEXT: v_lshr_b64 v[15:16], v[22:23], 16
-; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v51
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:660 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:668 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[45:46], v[35:36], 16
-; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v32
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_mov_b32_e32 v16, v37
-; SI-NEXT: v_mov_b32_e32 v46, v12
-; SI-NEXT: v_mov_b32_e32 v0, v50
+; SI-NEXT: v_lshr_b64 v[1:2], v[4:5], 16
; SI-NEXT: v_mov_b32_e32 v5, v9
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; SI-NEXT: v_mov_b32_e32 v14, v61
-; SI-NEXT: v_mov_b32_e32 v23, v43
-; SI-NEXT: v_lshr_b64 v[10:11], v[48:49], 16
-; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v26
-; SI-NEXT: v_mov_b32_e32 v11, v26
+; SI-NEXT: v_lshr_b64 v[3:4], v[35:36], 16
+; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v38
+; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v15
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v39, 0xffff0000, v5
+; SI-NEXT: v_lshr_b64 v[7:8], v[7:8], 16
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v12
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_lshr_b64 v[1:2], v[16:17], 16
+; SI-NEXT: v_mov_b32_e32 v58, v0
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v21
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v59, v1
+; SI-NEXT: v_lshr_b64 v[16:17], v[28:29], 16
+; SI-NEXT: v_lshr_b64 v[1:2], v[19:20], 16
+; SI-NEXT: v_mov_b32_e32 v17, v16
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[19:20], v[34:35], 16
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v55
+; SI-NEXT: v_lshr_b64 v[9:10], v[22:23], 16
; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v19, v8
-; SI-NEXT: v_lshr_b64 v[7:8], v[28:29], 16
-; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v16
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v11
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v1, v45
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v37
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(5)
+; SI-NEXT: v_mov_b32_e32 v8, v55
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v3, v37
+; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v4, v56
-; SI-NEXT: v_mov_b32_e32 v3, v55
-; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v4
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v42, 16, v3
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_and_b32_e32 v57, 0xffff0000, v20
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v22
-; SI-NEXT: v_lshr_b64 v[58:59], v[34:35], 16
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v55
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v2
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v38
+; SI-NEXT: v_mov_b32_e32 v10, v32
+; SI-NEXT: v_lshr_b64 v[40:41], v[48:49], 16
+; SI-NEXT: v_lshr_b64 v[49:50], v[13:14], 16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v13, v11
+; SI-NEXT: v_mov_b32_e32 v11, v26
+; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v24
+; SI-NEXT: v_mov_b32_e32 v41, v51
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v47
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v15, 0xffff0000, v22
; SI-NEXT: v_lshr_b64 v[55:56], v[27:28], 16
-; SI-NEXT: v_mov_b32_e32 v56, v4
-; SI-NEXT: s_waitcnt vmcnt(5) expcnt(4)
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v8
-; SI-NEXT: v_lshr_b64 v[36:37], v[25:26], 16
-; SI-NEXT: v_mov_b32_e32 v37, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v46
; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v2
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v39
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:676 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v47
-; SI-NEXT: v_mov_b32_e32 v47, v13
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v19
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:684 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v0
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, v61
+; SI-NEXT: v_lshr_b64 v[36:37], v[25:26], 16
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v0, v41
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v42
-; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v5
-; SI-NEXT: v_lshr_b64 v[49:50], v[31:32], 16
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v51
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v15
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v2, v8
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v22
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v24
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v62
-; SI-NEXT: v_mov_b32_e32 v17, v16
-; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v62
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v19, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v13
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v24, 16, v38
+; SI-NEXT: v_lshrrev_b32_e32 v38, 16, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v2
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v62, v10
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[44:45], v[31:32], 16
+; SI-NEXT: v_mov_b32_e32 v45, v1
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v14
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v52
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v34, 16, v2
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v46
+; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v2
+; SI-NEXT: v_mov_b32_e32 v2, v7
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, v40
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: v_lshr_b64 v[29:30], v[30:31], 16
; SI-NEXT: v_lshr_b64 v[31:32], v[33:34], 16
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v5
+; SI-NEXT: v_mov_b32_e32 v5, v4
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: v_lshr_b64 v[5:6], v[6:7], 16
; SI-NEXT: v_and_b32_e32 v30, 0xffff0000, v21
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v21
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, v49
+; SI-NEXT: v_mov_b32_e32 v47, v29
+; SI-NEXT: v_mov_b32_e32 v46, v28
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; SI-NEXT: v_mov_b32_e32 v26, v30
; SI-NEXT: v_mov_b32_e32 v27, v31
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v60
-; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v21
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v2
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v52
-; SI-NEXT: v_mov_b32_e32 v53, v29
-; SI-NEXT: v_mov_b32_e32 v52, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v20
-; SI-NEXT: v_lshrrev_b32_e32 v20, 16, v11
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v38
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[34:35], v[35:36], 16
-; SI-NEXT: v_mov_b32_e32 v51, v4
-; SI-NEXT: v_mov_b32_e32 v50, v3
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v14, v12
+; SI-NEXT: v_mov_b32_e32 v13, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v22
+; SI-NEXT: v_mov_b32_e32 v34, v47
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(2)
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v2, v19
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: v_lshr_b64 v[6:7], v[39:40], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[20:21], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v5, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[50:51], 16
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[46:47], 16
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[39:40], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v48, v58
+; SI-NEXT: v_mov_b32_e32 v49, v59
+; SI-NEXT: v_lshr_b64 v[11:12], v[11:12], 16
+; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v22, v4
-; SI-NEXT: v_mov_b32_e32 v21, v3
-; SI-NEXT: v_mov_b32_e32 v3, v47
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[11:12], v[48:49], 16
+; SI-NEXT: v_mov_b32_e32 v2, v44
+; SI-NEXT: v_lshr_b64 v[43:44], v[43:44], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshr_b64 v[20:21], v[6:7], 16
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[21:22], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[18:19], v[54:55], 16
+; SI-NEXT: v_mov_b32_e32 v40, v21
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshr_b64 v[58:59], v[6:7], 16
+; SI-NEXT: v_lshr_b64 v[6:7], v[15:16], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v45
-; SI-NEXT: v_lshr_b64 v[21:22], v[21:22], 16
-; SI-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v6, v36
+; SI-NEXT: v_mov_b32_e32 v16, v20
+; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[35:36], v[35:36], 16
+; SI-NEXT: v_mov_b32_e32 v36, v27
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v22, v7
-; SI-NEXT: v_mov_b32_e32 v47, v36
-; SI-NEXT: v_lshr_b64 v[35:36], v[0:1], 16
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[6:7], v[46:47], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v58
-; SI-NEXT: v_lshr_b64 v[57:58], v[57:58], 16
-; SI-NEXT: v_lshr_b64 v[58:59], v[48:49], 16
-; SI-NEXT: v_mov_b32_e32 v59, v53
-; SI-NEXT: v_mov_b32_e32 v48, v0
-; SI-NEXT: v_lshr_b64 v[0:1], v[40:41], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[26:27], 16
+; SI-NEXT: v_lshr_b64 v[25:26], v[0:1], 16
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[42:43], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mov_b32_e32 v21, v10
-; SI-NEXT: v_lshr_b64 v[9:10], v[9:10], 16
-; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[56:57], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[9:10], v[44:45], 16
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[0:1], v[32:33], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v44, v23
-; SI-NEXT: v_mov_b32_e32 v23, v38
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v49
-; SI-NEXT: v_lshr_b64 v[49:50], v[6:7], 16
-; SI-NEXT: v_lshr_b64 v[6:7], v[54:55], 16
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[32:33], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[26:27], v[50:51], 16
+; SI-NEXT: v_mov_b32_e32 v51, v41
+; SI-NEXT: v_mov_b32_e32 v27, v39
+; SI-NEXT: v_lshr_b64 v[47:48], v[60:61], 16
+; SI-NEXT: v_mov_b32_e32 v39, v3
+; SI-NEXT: v_mov_b32_e32 v33, v18
+; SI-NEXT: s_waitcnt vmcnt(6) expcnt(0)
+; SI-NEXT: v_lshr_b64 v[6:7], v[1:2], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[53:54], v[52:53], 16
-; SI-NEXT: v_lshr_b64 v[51:52], v[26:27], 16
-; SI-NEXT: v_mov_b32_e32 v54, v27
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(12)
-; SI-NEXT: v_lshr_b64 v[0:1], v[45:46], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshr_b64 v[6:7], v[6:7], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[17:18], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt vmcnt(8)
-; SI-NEXT: v_mov_b32_e32 v52, v6
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshr_b64 v[26:27], v[6:7], 16
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[6:7], v[8:9], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[28:29], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[52:53], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[19:20], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[41:42], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[14:15], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[23:24], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[12:13], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[37:38], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[24:25], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[30:31], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[60:61], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[6:7], v[28:29], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[30:31], 16
-; SI-NEXT: v_mov_b32_e32 v46, v9
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v9, v28
-; SI-NEXT: v_mov_b32_e32 v3, v32
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v50, v6
-; SI-NEXT: v_lshr_b64 v[10:11], v[6:7], 16
-; SI-NEXT: v_mov_b32_e32 v11, v40
-; SI-NEXT: v_lshr_b64 v[40:41], v[38:39], 16
+; SI-NEXT: v_lshr_b64 v[6:7], v[3:4], 16
+; SI-NEXT: v_mov_b32_e32 v41, v43
+; SI-NEXT: v_lshr_b64 v[53:54], v[13:14], 16
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_branch .LBB101_3
; SI-NEXT: .LBB101_2:
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(4)
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(3)
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: v_mov_b32_e32 v11, v40
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:488 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
-; SI-NEXT: v_mov_b32_e32 v48, v41
; SI-NEXT: s_mov_b64 s[4:5], -1
-; SI-NEXT: v_mov_b32_e32 v14, v61
-; SI-NEXT: v_mov_b32_e32 v19, v6
-; SI-NEXT: ; implicit-def: $vgpr26
-; SI-NEXT: ; implicit-def: $vgpr22
-; SI-NEXT: ; implicit-def: $vgpr47
+; SI-NEXT: ; implicit-def: $vgpr25
; SI-NEXT: ; implicit-def: $vgpr55
-; SI-NEXT: ; implicit-def: $vgpr59
-; SI-NEXT: ; implicit-def: $vgpr54
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr10
+; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: ; implicit-def: $vgpr36
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr26
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr27
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr11
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr16
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: ; implicit-def: $vgpr0
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr40
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr41
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr46
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr57
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr58
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr35
+; SI-NEXT: ; implicit-def: $vgpr53
+; SI-NEXT: ; implicit-def: $vgpr33
+; SI-NEXT: ; implicit-def: $vgpr47
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr34
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr53
-; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr51
; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr8
-; SI-NEXT: ; implicit-def: $vgpr4
-; SI-NEXT: ; implicit-def: $vgpr5
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr21
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
-; SI-NEXT: ; implicit-def: $vgpr1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_mov_b32_e32 v17, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v60, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v9, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr49
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
+; SI-NEXT: ; implicit-def: $vgpr17
+; SI-NEXT: ; implicit-def: $vgpr2
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v0, v61
; SI-NEXT: .LBB101_3: ; %Flow
; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
; SI-NEXT: s_cselect_b32 s4, 1, 0
; SI-NEXT: s_cmp_lg_u32 s4, 1
; SI-NEXT: s_cbranch_scc1 .LBB101_5
; SI-NEXT: ; %bb.4: ; %cmp.true
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:480 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v45
-; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v23
-; SI-NEXT: s_waitcnt vmcnt(3) expcnt(0)
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v23
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v37
+; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v39
+; SI-NEXT: s_waitcnt expcnt(2)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v50
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v52
-; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v19
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
-; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v14
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v20
+; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v28
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff0000, v52
+; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v56
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v60
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v24
-; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v48
-; SI-NEXT: v_and_b32_e32 v28, 0xffff0000, v42
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; SI-NEXT: v_and_b32_e32 v11, 0xffff0000, v30
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v32
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v44
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v51
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:476 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v16
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v13, 0xffff0000, v13
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v45, 0x40c00000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v37
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v45
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v17
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v56
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:692 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; SI-NEXT: v_add_f32_e32 v34, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[43:44], v[34:35], 16
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:464 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[45:46], v[20:21], 16
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v50, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:684 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:680 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:676 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v59, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:672 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v41, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:668 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v61, 16, v17
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:664 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v16
; SI-NEXT: v_lshrrev_b32_e32 v26, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:660 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v28, 16, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v16
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_f32_e32 v22, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v17
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v30, 16, v17
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v44
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v36, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v17, 0xffff0000, v62
; SI-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
; SI-NEXT: v_lshrrev_b32_e32 v32, 16, v17
-; SI-NEXT: v_lshr_b64 v[43:44], v[34:35], 16
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v38, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v49, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v52, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v18, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v56, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v58, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v40, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v60, 0x40c00000, v16
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
+; SI-NEXT: v_add_f32_e32 v25, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v27, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
+; SI-NEXT: v_add_f32_e32 v29, 0x40c00000, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
; SI-NEXT: v_add_f32_e32 v31, 0x40c00000, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; SI-NEXT: v_add_f32_e32 v24, 0x40c00000, v16
+; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v16
; SI-NEXT: v_add_f32_e32 v16, 0x40c00000, v0
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:588 ; 4-byte Folded Reload
-; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v24
-; SI-NEXT: v_lshr_b64 v[47:48], v[16:17], 16
-; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v62
+; SI-NEXT: v_lshr_b64 v[15:16], v[16:17], 16
+; SI-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v30, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v54, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v1
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v30
-; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v54
+; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v62, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v55, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v2
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v62
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v55
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v2, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v27
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v3
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:576 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v20, 0x40c00000, v0
-; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v28
+; SI-NEXT: v_add_f32_e32 v3, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v4
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v20
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:572 ; 4-byte Folded Reload
-; SI-NEXT: v_lshr_b64 v[28:29], v[36:37], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; SI-NEXT: v_add_f32_e32 v21, 0x40c00000, v0
+; SI-NEXT: v_add_f32_e32 v4, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v5
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v21
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v5, 0x40c00000, v0
@@ -211676,12 +211012,11 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v30
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v6, 0x40c00000, v0
@@ -211689,10 +211024,10 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v7, 0x40c00000, v0
@@ -211700,10 +211035,10 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v8, 0x40c00000, v0
@@ -211711,10 +211046,13 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v8
+; SI-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
+; SI-NEXT: v_and_b32_e32 v3, 0xffff0000, v55
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v9, 0x40c00000, v0
@@ -211722,11 +211060,10 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v9
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:468 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v9, 0xffff0000, v9
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v10, 0x40c00000, v0
@@ -211734,10 +211071,10 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v10
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v0
@@ -211745,451 +211082,406 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v11
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v27, 0xffff0000, v11
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v20, 0xffff0000, v11
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v12, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v13
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v12
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:456 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[15:16], v[0:1], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v13, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v14
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[47:48], v[0:1], 16
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:492 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v48, 0xffff0000, v4
; SI-NEXT: v_and_b32_e32 v44, 0xffff0000, v13
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; SI-NEXT: v_add_f32_e32 v14, 0x40c00000, v0
; SI-NEXT: v_add_f32_e32 v0, 0x40c00000, v33
-; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v35
; SI-NEXT: v_lshr_b64 v[34:35], v[22:23], 16
+; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v9
; SI-NEXT: v_and_b32_e32 v33, 0xffff0000, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff0000, v10
-; SI-NEXT: v_lshr_b64 v[10:11], v[49:50], 16
-; SI-NEXT: v_and_b32_e32 v35, 0xffff0000, v8
-; SI-NEXT: v_and_b32_e32 v49, 0xffff0000, v7
-; SI-NEXT: v_lshr_b64 v[7:8], v[40:41], 16
+; SI-NEXT: v_lshr_b64 v[11:12], v[25:26], 16
; SI-NEXT: v_and_b32_e32 v42, 0xffff0000, v14
-; SI-NEXT: v_lshr_b64 v[13:14], v[38:39], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v13, v42
+; SI-NEXT: v_mov_b32_e32 v14, v43
+; SI-NEXT: v_mov_b32_e32 v55, v34
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v46
-; SI-NEXT: v_lshr_b64 v[45:46], v[45:46], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, v21
+; SI-NEXT: v_lshr_b64 v[21:22], v[36:37], 16
+; SI-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
+; SI-NEXT: v_lshr_b64 v[9:10], v[60:61], 16
+; SI-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v23
-; SI-NEXT: v_lshr_b64 v[22:23], v[25:26], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[23:24], v[38:39], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v37
-; SI-NEXT: v_lshr_b64 v[36:37], v[52:53], 16
-; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v21
-; SI-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; SI-NEXT: v_mov_b32_e32 v23, v6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[36:37], v[49:50], 16
+; SI-NEXT: v_and_b32_e32 v37, 0xffff0000, v7
+; SI-NEXT: v_lshr_b64 v[6:7], v[58:59], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v39
-; SI-NEXT: v_lshr_b64 v[38:39], v[56:57], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[38:39], v[18:19], 16
+; SI-NEXT: v_lshr_b64 v[17:18], v[56:57], 16
+; SI-NEXT: v_mov_b32_e32 v18, v5
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v50
-; SI-NEXT: v_lshr_b64 v[50:51], v[18:19], 16
-; SI-NEXT: v_and_b32_e32 v51, 0xffff0000, v62
-; SI-NEXT: v_lshr_b64 v[17:18], v[58:59], 16
-; SI-NEXT: v_and_b32_e32 v18, 0xffff0000, v20
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[49:50], v[40:41], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v53
-; SI-NEXT: v_lshr_b64 v[52:53], v[54:55], 16
-; SI-NEXT: v_mov_b32_e32 v54, v33
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[52:53], v[52:53], 16
+; SI-NEXT: v_mov_b32_e32 v53, v15
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v19
-; SI-NEXT: v_lshr_b64 v[19:20], v[60:61], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v19, v6
+; SI-NEXT: v_lshr_b64 v[6:7], v[29:30], 16
+; SI-NEXT: v_lshr_b64 v[4:5], v[27:28], 16
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v54
+; SI-NEXT: v_mov_b32_e32 v54, v33
+; SI-NEXT: v_lshr_b64 v[33:34], v[54:55], 16
+; SI-NEXT: v_mov_b32_e32 v34, v45
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v57
-; SI-NEXT: v_lshr_b64 v[57:58], v[35:36], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v59
-; SI-NEXT: v_lshr_b64 v[58:59], v[9:10], 16
-; SI-NEXT: v_mov_b32_e32 v59, v45
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[58:59], v[37:38], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v41
-; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:452 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[40:41], v[51:52], 16
+; SI-NEXT: v_lshr_b64 v[41:42], v[35:36], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v61
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v26
-; SI-NEXT: v_mov_b32_e32 v26, v47
-; SI-NEXT: v_lshr_b64 v[46:47], v[49:50], 16
-; SI-NEXT: v_mov_b32_e32 v47, v28
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v55
-; SI-NEXT: v_mov_b32_e32 v55, v34
-; SI-NEXT: v_lshr_b64 v[34:35], v[27:28], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v1, v28
+; SI-NEXT: v_lshr_b64 v[27:28], v[8:9], 16
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:228 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:232 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; SI-NEXT: v_and_b32_e32 v0, 0xffff0000, v62
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, v32
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:216 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[1:2], v[31:32], 16
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v24, v7
-; SI-NEXT: v_lshr_b64 v[6:7], v[3:4], 16
-; SI-NEXT: v_mov_b32_e32 v4, v52
-; SI-NEXT: v_mov_b32_e32 v8, v6
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, v1
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(2)
-; SI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[5:6], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[51:52], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:336 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:340 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[21:22], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:328 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:332 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v30
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v19
-; SI-NEXT: v_lshr_b64 v[51:52], v[42:43], 16
-; SI-NEXT: v_mov_b32_e32 v5, v22
-; SI-NEXT: v_mov_b32_e32 v21, v38
-; SI-NEXT: v_mov_b32_e32 v22, v13
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_lshr_b64 v[5:6], v[5:6], 16
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[24:25], v[0:1], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[18:19], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:344 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:348 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v24
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:432 ; 4-byte Folded Spill
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[23:24], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:352 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:356 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v11
+; SI-NEXT: v_lshr_b64 v[5:6], v[3:4], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:456 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:460 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:464 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v17
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[10:11], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:384 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[16:17], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:364 ; 4-byte Folded Spill
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshr_b64 v[0:1], v[37:38], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:368 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:372 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v9
+; SI-NEXT: v_lshr_b64 v[11:12], v[48:49], 16
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:448 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:452 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, v19
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:84 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, v50
-; SI-NEXT: v_lshr_b64 v[49:50], v[12:13], 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:440 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:444 ; 4-byte Folded Spill
+; SI-NEXT: v_lshr_b64 v[2:3], v[18:19], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:388 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, v36
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v17
+; SI-NEXT: v_lshr_b64 v[16:17], v[16:17], 16
+; SI-NEXT: v_mov_b32_e32 v17, v23
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_mov_b32_e32 v1, v10
-; SI-NEXT: v_mov_b32_e32 v10, v15
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: v_mov_b32_e32 v2, v38
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: v_lshr_b64 v[0:1], v[54:55], 16
-; SI-NEXT: v_lshr_b64 v[53:54], v[44:45], 16
-; SI-NEXT: v_mov_b32_e32 v54, v43
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:376 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:380 ; 4-byte Folded Spill
-; SI-NEXT: .LBB101_5: ; %end
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:392 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:400 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v4
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v5
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v2, v52
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:480 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:484 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:488 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(1)
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_mov_b32_e32 v2, v36
+; SI-NEXT: v_lshr_b64 v[35:36], v[20:21], 16
+; SI-NEXT: v_mov_b32_e32 v36, v14
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:468 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:472 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:476 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v26
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:440 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:444 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v3, 0xffff, v8
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v23, 0xffff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v58
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
+; SI-NEXT: v_lshr_b64 v[2:3], v[22:23], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:412 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: v_mov_b32_e32 v2, v21
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:420 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:428 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[2:3], v[44:45], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:396 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:400 ; 4-byte Folded Spill
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_lshr_b64 v[2:3], v[13:14], 16
+; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:408 ; 4-byte Folded Spill
+; SI-NEXT: .LBB101_5: ; %end
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:436 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(14) expcnt(2)
+; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v49
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: v_and_b32_e32 v23, 0xffff, v17
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v58
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v40
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v41
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v33
; SI-NEXT: v_readlane_b32 s30, v63, 3
; SI-NEXT: v_readlane_b32 s31, v63, 4
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v12
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:428 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:432 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt expcnt(1)
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:396 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(9) expcnt(0)
+; SI-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:460 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v21
-; SI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:412 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:416 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:420 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v21, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:484 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v0
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:472 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v21, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v25
; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v20
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v57
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v35
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:336 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(7)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v26
; SI-NEXT: v_or_b32_e32 v2, v2, v4
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; SI-NEXT: v_or_b32_e32 v3, v3, v4
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v35
+; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v6, v5
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:328 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: v_or_b32_e32 v6, v6, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v17
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v7, v7, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v8
+; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v27
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v39
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v55
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v39
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:360 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:364 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_or_b32_e32 v12, v12, v14
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v25
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:368 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:372 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v40
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: v_or_b32_e32 v14, v14, v16
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v46
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v16, v16, v17
+; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; SI-NEXT: v_or_b32_e32 v17, v18, v17
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:332 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
; SI-NEXT: v_or_b32_e32 v18, v18, v20
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:340 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
; SI-NEXT: v_or_b32_e32 v20, v20, v22
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; SI-NEXT: v_or_b32_e32 v21, v21, v22
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v49
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
; SI-NEXT: v_or_b32_e32 v22, v22, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v25
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; SI-NEXT: v_or_b32_e32 v23, v23, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:356 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v34
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
; SI-NEXT: v_or_b32_e32 v24, v24, v25
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v47
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:424 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v35
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:404 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:408 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:376 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:380 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(5)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
-; SI-NEXT: s_waitcnt vmcnt(3)
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v27
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v53
; SI-NEXT: v_or_b32_e32 v26, v26, v29
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v55
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v35
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_or_b32_e32 v27, v27, v29
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:384 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:388 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v28
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v53
-; SI-NEXT: v_or_b32_e32 v28, v29, v28
-; SI-NEXT: v_and_b32_e32 v29, 0xffff, v59
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v47
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -212205,19 +211497,17 @@ define inreg <64 x half> @bitcast_v64bf16_to_v64f16_scalar(<64 x bfloat> inreg %
; SI-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v35
+; SI-NEXT: v_or_b32_e32 v28, v29, v28
+; SI-NEXT: v_and_b32_e32 v29, 0xffff, v34
; SI-NEXT: v_or_b32_e32 v29, v29, v30
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v31, 16, v51
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
; SI-NEXT: v_or_b32_e32 v30, v30, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v54
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v36
; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
-; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:688 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:696 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -222258,12 +221548,12 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_lshr_b64 v[18:19], v[60:61], 16
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:264 ; 4-byte Folded Spill
-; SI-NEXT: v_mov_b32_e32 v46, v45
; SI-NEXT: v_mov_b32_e32 v39, v50
; SI-NEXT: v_lshr_b64 v[22:23], v[4:5], 16
; SI-NEXT: v_lshr_b64 v[49:50], v[2:3], 16
; SI-NEXT: v_lshr_b64 v[32:33], v[0:1], 16
; SI-NEXT: v_mov_b32_e32 v59, v58
+; SI-NEXT: v_mov_b32_e32 v46, v45
; SI-NEXT: v_mov_b32_e32 v36, v35
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[18:19], v[16:17], 16
@@ -222296,7 +221586,6 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_lshr_b64 v[18:19], v[44:45], 16
; SI-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; SI-NEXT: s_waitcnt expcnt(0)
@@ -222304,191 +221593,167 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; SI-NEXT: .LBB105_5: ; %end
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v1
; SI-NEXT: s_waitcnt expcnt(1)
; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:244 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_and_b32_e32 v2, 0xffff, v24
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v21
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v5
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v61
-; SI-NEXT: s_waitcnt vmcnt(6)
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v17
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v7
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v15
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v9
; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_and_b32_e32 v8, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v7
+; SI-NEXT: s_waitcnt vmcnt(4)
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v13
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v11
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v22
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v49
+; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v32
+; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v28
; SI-NEXT: v_readlane_b32 s30, v63, 3
; SI-NEXT: v_readlane_b32 s31, v63, 4
; SI-NEXT: v_readlane_b32 s36, v63, 2
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(13)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v42
; SI-NEXT: v_or_b32_e32 v1, v2, v1
; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(12)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; SI-NEXT: s_waitcnt vmcnt(11)
+; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-NEXT: s_waitcnt vmcnt(10)
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-NEXT: s_waitcnt vmcnt(9)
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-NEXT: s_waitcnt vmcnt(8)
+; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v27
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v56
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v30
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SI-NEXT: v_or_b32_e32 v2, v2, v3
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v31
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v31
+; SI-NEXT: v_and_b32_e32 v31, 0xffff, v36
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; SI-NEXT: v_or_b32_e32 v5, v6, v5
; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
; SI-NEXT: v_or_b32_e32 v6, v6, v7
; SI-NEXT: v_lshlrev_b32_e32 v7, 16, v48
; SI-NEXT: v_or_b32_e32 v7, v8, v7
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v44
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v8, v8, v9
; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v37
; SI-NEXT: v_or_b32_e32 v9, v10, v9
; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v11
+; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v25
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:176 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; SI-NEXT: v_or_b32_e32 v11, v12, v11
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:172 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_or_b32_e32 v12, v12, v14
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v54
; SI-NEXT: v_or_b32_e32 v13, v13, v14
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: v_or_b32_e32 v14, v14, v16
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v47
; SI-NEXT: v_or_b32_e32 v15, v15, v16
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
; SI-NEXT: v_or_b32_e32 v16, v16, v18
; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v39
; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v18, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
; SI-NEXT: v_or_b32_e32 v18, v18, v20
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v55
; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v24
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
; SI-NEXT: v_or_b32_e32 v20, v20, v22
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v53
; SI-NEXT: v_or_b32_e32 v21, v21, v22
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v32
+; SI-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v40
; SI-NEXT: v_or_b32_e32 v23, v23, v24
; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:164 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:168 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v24
; SI-NEXT: v_or_b32_e32 v24, v24, v25
; SI-NEXT: v_and_b32_e32 v25, 0xffff, v59
-; SI-NEXT: s_waitcnt vmcnt(4)
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v29
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v25, v25, v26
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v31
-; SI-NEXT: v_and_b32_e32 v31, 0xffff, v36
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v28
-; SI-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:152 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v27
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v26
; SI-NEXT: v_or_b32_e32 v26, v26, v28
-; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v56
+; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v29
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v28
; SI-NEXT: v_or_b32_e32 v27, v27, v28
; SI-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:156 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:160 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v30, 16, v33
; SI-NEXT: v_and_b32_e32 v28, 0xffff, v28
-; SI-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; SI-NEXT: v_or_b32_e32 v28, v28, v29
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v46
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -222509,10 +221774,9 @@ define inreg <64 x i16> @bitcast_v64bf16_to_v64i16_scalar(<64 x bfloat> inreg %a
; SI-NEXT: v_or_b32_e32 v29, v29, v30
; SI-NEXT: v_and_b32_e32 v30, 0xffff, v38
; SI-NEXT: v_or_b32_e32 v30, v30, v32
-; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v33
+; SI-NEXT: v_lshlrev_b32_e32 v32, 16, v32
; SI-NEXT: v_or_b32_e32 v31, v31, v32
; SI-NEXT: s_or_saveexec_b64 s[4:5], -1
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
@@ -228697,20 +227961,24 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v56
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v6, v8, v6
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v15
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v15
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v13
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v8, v10, v8
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v57
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v45
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v17
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v53
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v26
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v22
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v19
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v36
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v21
; SI-NEXT: v_and_b32_e32 v27, 0xffff, v55
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
@@ -228723,12 +227991,10 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_readlane_b32 s35, v63, 1
; SI-NEXT: v_readlane_b32 s34, v63, 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
; SI-NEXT: v_or_b32_e32 v9, v9, v10
; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v44
; SI-NEXT: v_or_b32_e32 v10, v12, v10
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v57
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v34
; SI-NEXT: v_or_b32_e32 v11, v11, v12
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v42
@@ -228736,37 +228002,30 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v35
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v15, 0xffff, v53
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: v_or_b32_e32 v14, v16, v14
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v33
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v22
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v36
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_or_b32_e32 v16, v18, v16
; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v47
; SI-NEXT: v_or_b32_e32 v17, v17, v18
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v20
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v33
+; SI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_or_b32_e32 v18, v18, v20
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v41
; SI-NEXT: v_or_b32_e32 v19, v19, v20
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v48
; SI-NEXT: v_or_b32_e32 v20, v20, v22
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v23
+; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; SI-NEXT: v_or_b32_e32 v21, v21, v22
; SI-NEXT: v_and_b32_e32 v22, 0xffff, v24
-; SI-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v23
; SI-NEXT: v_or_b32_e32 v22, v22, v24
@@ -228775,7 +228034,6 @@ define inreg <64 x i16> @bitcast_v64f16_to_v64i16_scalar(<64 x half> inreg %a, i
; SI-NEXT: v_or_b32_e32 v23, v23, v24
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v25
; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v25
; SI-NEXT: v_or_b32_e32 v24, v24, v26
@@ -230905,22 +230163,14 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: .LBB111_3: ; %Flow
; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
-; SI-NEXT: v_readlane_b32 s66, v33, 10
-; SI-NEXT: v_readlane_b32 s67, v33, 11
-; SI-NEXT: s_mov_b32 s22, s66
-; SI-NEXT: v_readlane_b32 s66, v33, 12
-; SI-NEXT: v_readlane_b32 s67, v33, 13
-; SI-NEXT: s_mov_b32 s24, s66
-; SI-NEXT: v_readlane_b32 s66, v33, 14
-; SI-NEXT: v_readlane_b32 s67, v33, 15
-; SI-NEXT: s_mov_b32 s26, s66
-; SI-NEXT: v_readlane_b32 s66, v33, 16
; SI-NEXT: s_cselect_b32 s55, 1, 0
-; SI-NEXT: v_readlane_b32 s67, v33, 17
; SI-NEXT: s_cmp_lg_u32 s55, 1
-; SI-NEXT: s_mov_b32 s28, s66
-; SI-NEXT: v_readlane_b32 s66, v33, 18
-; SI-NEXT: v_readlane_b32 s67, v33, 19
+; SI-NEXT: v_readlane_b32 s55, v33, 18
+; SI-NEXT: v_readlane_b32 s22, v33, 10
+; SI-NEXT: v_readlane_b32 s24, v33, 12
+; SI-NEXT: v_readlane_b32 s26, v33, 14
+; SI-NEXT: v_readlane_b32 s28, v33, 16
+; SI-NEXT: s_mov_b32 s66, s55
; SI-NEXT: s_cbranch_scc1 .LBB111_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s20, s20, 3
@@ -231173,18 +230423,17 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_or_b32 s24, s24, s25
; SI-NEXT: s_and_b32 s25, s59, 0xffff
; SI-NEXT: s_lshl_b32 s26, s35, 16
-; SI-NEXT: v_readlane_b32 s28, v33, 20
+; SI-NEXT: v_readlane_b32 s27, v33, 20
; SI-NEXT: s_or_b32 s25, s25, s26
; SI-NEXT: s_and_b32 s26, s56, 0xffff
-; SI-NEXT: s_lshl_b32 s27, s28, 16
-; SI-NEXT: v_readlane_b32 s29, v33, 21
+; SI-NEXT: s_lshl_b32 s27, s27, 16
; SI-NEXT: s_or_b32 s26, s26, s27
; SI-NEXT: s_and_b32 s27, s57, 0xffff
; SI-NEXT: s_lshl_b32 s28, s37, 16
-; SI-NEXT: v_readlane_b32 s56, v33, 22
+; SI-NEXT: v_readlane_b32 s29, v33, 22
; SI-NEXT: s_or_b32 s27, s27, s28
; SI-NEXT: s_and_b32 s28, s46, 0xffff
-; SI-NEXT: s_lshl_b32 s29, s56, 16
+; SI-NEXT: s_lshl_b32 s29, s29, 16
; SI-NEXT: s_or_b32 s28, s28, s29
; SI-NEXT: s_and_b32 s29, s47, 0xffff
; SI-NEXT: s_lshl_b32 s46, s39, 16
@@ -231192,7 +230441,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_readlane_b32 s46, v33, 24
; SI-NEXT: s_and_b32 s44, s44, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 25
; SI-NEXT: s_or_b32 s44, s44, s46
; SI-NEXT: s_and_b32 s45, s45, 0xffff
; SI-NEXT: s_lshl_b32 s46, s83, 16
@@ -231200,7 +230448,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_readlane_b32 s46, v33, 26
; SI-NEXT: s_and_b32 s42, s42, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 27
; SI-NEXT: s_or_b32 s42, s42, s46
; SI-NEXT: s_and_b32 s43, s43, 0xffff
; SI-NEXT: s_lshl_b32 s46, s84, 16
@@ -231208,7 +230455,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_readlane_b32 s46, v33, 28
; SI-NEXT: s_and_b32 s40, s40, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 29
; SI-NEXT: s_or_b32 s40, s40, s46
; SI-NEXT: s_and_b32 s41, s41, 0xffff
; SI-NEXT: s_lshl_b32 s46, s85, 16
@@ -231216,7 +230462,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_readlane_b32 s46, v33, 30
; SI-NEXT: s_and_b32 s14, s14, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 31
; SI-NEXT: s_or_b32 s14, s14, s46
; SI-NEXT: s_and_b32 s15, s15, 0xffff
; SI-NEXT: s_lshl_b32 s46, s86, 16
@@ -231224,7 +230469,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: v_readlane_b32 s46, v33, 32
; SI-NEXT: s_and_b32 s12, s12, 0xffff
; SI-NEXT: s_lshl_b32 s46, s46, 16
-; SI-NEXT: v_readlane_b32 s47, v33, 33
; SI-NEXT: s_or_b32 s12, s12, s46
; SI-NEXT: s_and_b32 s13, s13, 0xffff
; SI-NEXT: s_lshl_b32 s46, s87, 16
@@ -231255,8 +230499,6 @@ define inreg <64 x half> @bitcast_v64i16_to_v64f16_scalar(<64 x i16> inreg %a, i
; SI-NEXT: s_lshl_b32 s46, s99, 16
; SI-NEXT: s_or_b32 s5, s5, s46
; SI-NEXT: v_readlane_b32 s30, v32, 34
-; SI-NEXT: v_readlane_b32 s57, v33, 23
-; SI-NEXT: v_readlane_b32 s47, v33, 35
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_mov_b32_e32 v2, s18
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 78585c9150ddb..ed0ec2678b57d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -25987,7 +25987,6 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:44
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v3, v3, v34, s4
; VI-NEXT: v_perm_b32 v1, v1, v29, s4
; VI-NEXT: v_readlane_b32 s30, v63, 16
@@ -26008,7 +26007,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_readlane_b32 s36, v63, 2
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v5, v33, v5, s4
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_or_b32_e32 v3, v3, v5
@@ -26019,8 +26018,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; VI-NEXT: v_or_b32_e32 v3, v3, v4
; VI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:52
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v3, v28, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -26403,8 +26401,6 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_or_b32_e32 v5, v5, v6
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
; GFX9-NEXT: v_readlane_b32 s30, v63, 12
@@ -26421,7 +26417,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v33, v5, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
@@ -26432,9 +26428,7 @@ define inreg <64 x i8> @bitcast_v16f32_to_v64i8_scalar(<16 x float> inreg %a, i3
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v28, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
@@ -53547,7 +53541,6 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_or_b32_e32 v5, v5, v6
; VI-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:36
; VI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v3, v3, v36, s4
; VI-NEXT: v_perm_b32 v1, v1, v31, s4
; VI-NEXT: v_readlane_b32 s30, v63, 16
@@ -53568,7 +53561,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_readlane_b32 s36, v63, 2
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v5, v35, v5, s4
; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-NEXT: v_or_b32_e32 v3, v3, v5
@@ -53579,8 +53572,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_or_b32_e32 v3, v3, v4
; VI-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44
; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v3, v30, v3, s4
; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; VI-NEXT: v_or_b32_e32 v1, v1, v3
@@ -53591,9 +53583,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: v_perm_b32 v1, v11, v27, s4
-; VI-NEXT: s_waitcnt vmcnt(1)
+; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_perm_b32 v2, v26, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
@@ -53964,8 +53955,6 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v5, v5, v6
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:36
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v36, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v31, s4
; GFX9-NEXT: v_readlane_b32 s30, v63, 12
@@ -53982,7 +53971,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v35, v5, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
@@ -53993,9 +53982,7 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v30, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
@@ -54006,9 +53993,8 @@ define inreg <64 x i8> @bitcast_v8f64_to_v64i8_scalar(<8 x double> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v1, v11, v27, s4
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v2, v26, v2, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX9-NEXT: v_or_b32_e32 v1, v1, v2
@@ -65429,10 +65415,10 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v2, v33, v2, s4
; VI-NEXT: v_perm_b32 v1, v47, v1, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v2, v33, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:48
@@ -65447,10 +65433,10 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) #0 {
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v2, v32, v2, s4
; VI-NEXT: v_perm_b32 v1, v18, v1, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v2, v32, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:56
@@ -66473,8 +66459,8 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_and_b64 s[66:67], s[66:67], exec
; SI-NEXT: s_cselect_b32 vcc_hi, 1, 0
; SI-NEXT: s_cmp_lg_u32 vcc_hi, 1
-; SI-NEXT: v_readlane_b32 s66, v5, 4
-; SI-NEXT: v_readlane_b32 s67, v5, 5
+; SI-NEXT: v_readlane_b32 vcc_hi, v5, 4
+; SI-NEXT: s_mov_b32 s66, vcc_hi
; SI-NEXT: s_cbranch_scc1 .LBB97_5
; SI-NEXT: ; %bb.4: ; %cmp.true
; SI-NEXT: s_add_i32 s58, s58, 3
@@ -66612,13 +66598,12 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_lshr_b32 s45, s5, 8
; SI-NEXT: v_writelane_b32 v5, s17, 7
; SI-NEXT: .LBB97_5: ; %end
-; SI-NEXT: v_readlane_b32 s18, v5, 0
-; SI-NEXT: v_readlane_b32 s19, v5, 1
+; SI-NEXT: v_readlane_b32 s17, v5, 0
; SI-NEXT: s_and_b32 s16, s42, 0xff
-; SI-NEXT: s_lshl_b32 s17, s18, 8
-; SI-NEXT: v_readlane_b32 s18, v5, 2
+; SI-NEXT: s_lshl_b32 s17, s17, 8
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: s_and_b32 s17, s18, 0xff
+; SI-NEXT: v_readlane_b32 s17, v5, 2
+; SI-NEXT: s_and_b32 s17, s17, 0xff
; SI-NEXT: s_lshl_b32 s17, s17, 16
; SI-NEXT: s_lshl_b32 s18, s66, 24
; SI-NEXT: s_and_b32 s16, s16, 0xffff
@@ -66634,7 +66619,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_and_b32 s16, s16, 0xffff
; SI-NEXT: s_or_b32 s17, s18, s17
; SI-NEXT: s_or_b32 s16, s16, s17
-; SI-NEXT: v_readlane_b32 s19, v5, 3
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s16
@@ -66808,7 +66792,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v1, s4
; SI-NEXT: v_readlane_b32 s30, v4, 34
-; SI-NEXT: v_readlane_b32 s19, v5, 7
; SI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:60
; SI-NEXT: v_readlane_b32 s31, v4, 35
; SI-NEXT: v_readlane_b32 s99, v4, 33
@@ -67606,8 +67589,6 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v5, v5, v6
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
; GFX9-NEXT: v_readlane_b32 s30, v63, 12
@@ -67624,7 +67605,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v33, v5, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
@@ -67635,9 +67616,7 @@ define inreg <64 x i8> @bitcast_v32i16_to_v64i8_scalar(<32 x i16> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v28, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
@@ -75223,9 +75202,6 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_and_b32_e32 v50, 0xffff0000, v28
; SI-NEXT: v_lshrrev_b32_e32 v39, 16, v44
; SI-NEXT: v_lshr_b64 v[24:25], v[38:39], 16
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v14
; SI-NEXT: v_mov_b32_e32 v55, v14
; SI-NEXT: v_lshrrev_b32_e32 v37, 16, v20
@@ -75242,6 +75218,8 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v40
; SI-NEXT: v_lshrrev_b32_e32 v46, 16, v17
; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: v_lshrrev_b32_e32 v35, 16, v0
@@ -75489,20 +75467,18 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v9
; SI-NEXT: v_lshr_b64 v[55:56], v[1:2], 16
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: v_and_b32_e32 v56, 0xffff0000, v4
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v14
; SI-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v1
; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; SI-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; SI-NEXT: v_lshr_b64 v[15:16], v[11:12], 16
; SI-NEXT: s_waitcnt expcnt(1)
@@ -75580,23 +75556,24 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; SI-NEXT: .LBB103_5: ; %end
+; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; SI-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v24
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v31
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v52
+; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v11
; SI-NEXT: v_lshlrev_b32_e32 v6, 16, v49
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v22
-; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v37
+; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v50
+; SI-NEXT: v_and_b32_e32 v11, 0xffff, v29
+; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v25
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v53
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; SI-NEXT: s_waitcnt vmcnt(2)
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v1
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v35
@@ -75606,44 +75583,35 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v61
; SI-NEXT: v_or_b32_e32 v3, v4, v3
; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(2)
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v18
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v17
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; SI-NEXT: v_or_b32_e32 v4, v4, v5
; SI-NEXT: v_and_b32_e32 v5, 0xffff, v57
; SI-NEXT: v_or_b32_e32 v5, v5, v6
; SI-NEXT: v_and_b32_e32 v6, 0xffff, v54
; SI-NEXT: v_or_b32_e32 v6, v6, v8
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
-; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
; SI-NEXT: v_lshlrev_b32_e32 v8, 16, v39
; SI-NEXT: v_or_b32_e32 v7, v7, v8
; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v9, 16, v18
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
; SI-NEXT: v_or_b32_e32 v8, v8, v9
-; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v11, 0xffff, v29
-; SI-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; SI-NEXT: v_lshlrev_b32_e32 v10, 16, v37
+; SI-NEXT: v_and_b32_e32 v9, 0xffff, v9
; SI-NEXT: v_or_b32_e32 v9, v9, v10
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v12
-; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v50
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
; SI-NEXT: v_or_b32_e32 v10, v10, v12
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v63
; SI-NEXT: v_or_b32_e32 v11, v11, v12
; SI-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v63, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
@@ -75661,17 +75629,15 @@ define inreg <32 x half> @bitcast_v32bf16_to_v32f16_scalar(<32 x bfloat> inreg %
; SI-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(14)
-; SI-NEXT: v_lshlrev_b32_e32 v13, 16, v25
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
; SI-NEXT: v_or_b32_e32 v12, v12, v13
; SI-NEXT: v_and_b32_e32 v13, 0xffff, v27
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v55
; SI-NEXT: v_or_b32_e32 v14, v14, v16
-; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: s_setpc_b64 s[30:31]
;
@@ -79351,10 +79317,13 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_bfe_u32 v7, v1, 8, 8
; SI-NEXT: s_branch .LBB105_6
; SI-NEXT: .LBB105_5:
-; SI-NEXT: v_readlane_b32 s18, v19, 4
-; SI-NEXT: v_readlane_b32 s20, v19, 2
-; SI-NEXT: v_readlane_b32 s22, v19, 6
-; SI-NEXT: v_readlane_b32 s24, v19, 0
+; SI-NEXT: v_readlane_b32 s16, v19, 4
+; SI-NEXT: s_mov_b32 s18, s16
+; SI-NEXT: v_readlane_b32 s16, v19, 2
+; SI-NEXT: s_mov_b32 s20, s16
+; SI-NEXT: v_readlane_b32 s16, v19, 6
+; SI-NEXT: s_mov_b32 s22, s16
+; SI-NEXT: v_readlane_b32 s16, v19, 0
; SI-NEXT: v_mov_b32_e32 v1, s99
; SI-NEXT: v_mov_b32_e32 v2, s96
; SI-NEXT: v_mov_b32_e32 v3, s86
@@ -79371,10 +79340,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; SI-NEXT: v_mov_b32_e32 v14, s89
; SI-NEXT: v_mov_b32_e32 v15, s79
; SI-NEXT: v_mov_b32_e32 v16, s77
-; SI-NEXT: v_readlane_b32 s19, v19, 5
-; SI-NEXT: v_readlane_b32 s21, v19, 3
-; SI-NEXT: v_readlane_b32 s23, v19, 7
-; SI-NEXT: v_readlane_b32 s25, v19, 1
+; SI-NEXT: s_mov_b32 s24, s16
; SI-NEXT: .LBB105_6: ; %end
; SI-NEXT: s_and_b32 s16, s42, 0xff
; SI-NEXT: s_lshl_b32 s17, s24, 8
@@ -80012,7 +79978,6 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:52
; VI-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; VI-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; VI-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; VI-NEXT: v_readlane_b32 s30, v63, 16
; VI-NEXT: v_readlane_b32 s31, v63, 17
; VI-NEXT: v_readlane_b32 s65, v63, 15
@@ -80032,8 +79997,9 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; VI-NEXT: v_readlane_b32 s35, v63, 1
; VI-NEXT: v_readlane_b32 s34, v63, 0
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_perm_b32 v2, v26, v2, s4
; VI-NEXT: v_perm_b32 v1, v36, v1, s4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_perm_b32 v2, v26, v2, s4
; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: v_perm_b32 v2, v19, v9, s4
@@ -80416,8 +80382,6 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v5, v5, v6
; GFX9-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:44
; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-NEXT: v_perm_b32 v3, v3, v34, s4
; GFX9-NEXT: v_perm_b32 v1, v1, v29, s4
; GFX9-NEXT: v_readlane_b32 s30, v63, 12
@@ -80434,7 +80398,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_readlane_b32 s36, v63, 2
; GFX9-NEXT: v_readlane_b32 s35, v63, 1
; GFX9-NEXT: v_readlane_b32 s34, v63, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v5, v33, v5, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX9-NEXT: v_or_b32_e32 v3, v3, v5
@@ -80445,9 +80409,7 @@ define inreg <64 x i8> @bitcast_v32f16_to_v64i8_scalar(<32 x half> inreg %a, i32
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
; GFX9-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:52
; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_perm_b32 v3, v28, v3, s4
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
index d3bb79fe0084e..d590470cd2026 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll
@@ -53351,32 +53351,33 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v12, 16, v54
; SI-NEXT: v_and_b32_e32 v14, 0xffff, v59
; SI-NEXT: v_or_b32_e32 v12, v14, v12
-; SI-NEXT: v_and_b32_e32 v13, 0xffff, v47
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v22
+; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v47
; SI-NEXT: v_or_b32_e32 v13, v13, v14
; SI-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; SI-NEXT: s_waitcnt vmcnt(7)
+; SI-NEXT: s_waitcnt vmcnt(8)
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v55
; SI-NEXT: v_or_b32_e32 v14, v16, v14
; SI-NEXT: v_and_b32_e32 v15, 0xffff, v41
; SI-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; SI-NEXT: v_or_b32_e32 v15, v15, v16
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v33
-; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v22
; SI-NEXT: v_or_b32_e32 v16, v16, v18
; SI-NEXT: v_and_b32_e32 v17, 0xffff, v19
; SI-NEXT: v_lshlrev_b32_e32 v18, 16, v39
; SI-NEXT: v_or_b32_e32 v17, v17, v18
-; SI-NEXT: s_waitcnt vmcnt(6)
+; SI-NEXT: s_waitcnt vmcnt(7)
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v21
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v20
; SI-NEXT: v_or_b32_e32 v18, v18, v20
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v36
; SI-NEXT: v_lshlrev_b32_e32 v20, 16, v37
; SI-NEXT: v_or_b32_e32 v19, v19, v20
-; SI-NEXT: s_waitcnt vmcnt(5)
+; SI-NEXT: s_waitcnt vmcnt(6)
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v43
-; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v25
; SI-NEXT: v_or_b32_e32 v20, v20, v22
; SI-NEXT: v_and_b32_e32 v21, 0xffff, v50
; SI-NEXT: v_lshlrev_b32_e32 v22, 16, v23
@@ -53385,26 +53386,24 @@ define inreg <60 x i16> @bitcast_v60f16_to_v60i16_scalar(<60 x half> inreg %a, i
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v42
; SI-NEXT: v_or_b32_e32 v22, v22, v24
; SI-NEXT: v_and_b32_e32 v23, 0xffff, v30
-; SI-NEXT: s_waitcnt vmcnt(4)
+; SI-NEXT: s_waitcnt vmcnt(5)
; SI-NEXT: v_lshlrev_b32_e32 v24, 16, v27
; SI-NEXT: v_or_b32_e32 v23, v23, v24
; SI-NEXT: v_and_b32_e32 v24, 0xffff, v51
; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v26
; SI-NEXT: v_or_b32_e32 v24, v24, v26
-; SI-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; SI-NEXT: v_and_b32_e32 v27, 0xffff, v44
+; SI-NEXT: s_waitcnt vmcnt(3)
+; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v28
; SI-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; SI-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v60
+; SI-NEXT: v_and_b32_e32 v27, 0xffff, v44
; SI-NEXT: v_and_b32_e32 v29, 0xffff, v29
; SI-NEXT: v_readlane_b32 s30, v63, 0
; SI-NEXT: v_readlane_b32 s31, v63, 1
-; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_and_b32_e32 v25, 0xffff, v26
-; SI-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; SI-NEXT: s_waitcnt vmcnt(1)
+; SI-NEXT: v_and_b32_e32 v25, 0xffff, v25
; SI-NEXT: v_or_b32_e32 v25, v25, v26
; SI-NEXT: v_and_b32_e32 v26, 0xffff, v48
-; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v60
; SI-NEXT: v_or_b32_e32 v26, v26, v28
; SI-NEXT: v_lshlrev_b32_e32 v28, 16, v45
; SI-NEXT: buffer_load_dword v62, off, s[0:3], s32 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
index 3f034d709b102..d37921faf055c 100644
--- a/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
+++ b/llvm/test/CodeGen/AMDGPU/dummy-regalloc-priority-advisor.mir
@@ -27,7 +27,7 @@ body: |
; DEFAULT-NEXT: SI_SPILL_V128_SAVE $vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5)
; DEFAULT-NEXT: SI_SPILL_V32_SAVE $vgpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5)
; DEFAULT-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
- ; DEFAULT-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
+ ; DEFAULT-NEXT: renamable $vgpr2 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
; DEFAULT-NEXT: renamable $vgpr3 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
; DEFAULT-NEXT: renamable $vgpr3 = V_ADD_U32_e32 killed $vgpr2, killed $vgpr3, implicit $exec
; DEFAULT-NEXT: SI_RETURN implicit $vgpr3, implicit $vgpr0, implicit $vgpr1
@@ -39,7 +39,7 @@ body: |
; DUMMY-NEXT: SI_SPILL_V32_SAVE $vgpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5)
; DUMMY-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
; DUMMY-NEXT: renamable $vgpr2 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
- ; DUMMY-NEXT: renamable $vgpr3_vgpr4_vgpr5_vgpr6 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
+ ; DUMMY-NEXT: renamable $vgpr3 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
; DUMMY-NEXT: renamable $vgpr3 = V_ADD_U32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
; DUMMY-NEXT: SI_RETURN implicit $vgpr3, implicit $vgpr0, implicit $vgpr1
undef %1.sub0:vreg_128 = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index f805b11d35db5..06a52198cad32 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -3366,21 +3366,26 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_add_i32 s2, s32, 16
; GFX11-NEXT: v_mov_b32_e32 v29, v43
; GFX11-NEXT: scratch_store_b128 off, v[50:53], s2
-; GFX11-NEXT: s_clause 0x3 ; 64-byte Folded Reload
-; GFX11-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1584
+; GFX11-NEXT: s_clause 0x3 ; 60-byte Folded Reload
+; GFX11-NEXT: scratch_load_b96 v[0:2], off, s33 offset:1588
; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536
; GFX11-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1568
; GFX11-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1552
; GFX11-NEXT: s_add_i32 s2, s33, 0x400
; GFX11-NEXT: v_dual_mov_b32 v30, v44 :: v_dual_mov_b32 v31, v45
-; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 42
+; GFX11-NEXT: s_waitcnt vmcnt(6)
; GFX11-NEXT: v_dual_mov_b32 v17, v20 :: v_dual_mov_b32 v18, v21
-; GFX11-NEXT: v_dual_mov_b32 v19, v22 :: v_dual_mov_b32 v20, v23
-; GFX11-NEXT: v_dual_mov_b32 v21, v58 :: v_dual_mov_b32 v22, v59
-; GFX11-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v24, v61
-; GFX11-NEXT: v_dual_mov_b32 v25, v54 :: v_dual_mov_b32 v26, v55
+; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: v_dual_mov_b32 v20, v23 :: v_dual_mov_b32 v21, v58
+; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: v_dual_mov_b32 v24, v61 :: v_dual_mov_b32 v25, v54
+; GFX11-NEXT: v_dual_mov_b32 v19, v22 :: v_dual_mov_b32 v22, v59
+; GFX11-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v26, v55
; GFX11-NEXT: v_dual_mov_b32 v27, v56 :: v_dual_mov_b32 v28, v57
+; GFX11-NEXT: s_waitcnt vmcnt(3)
+; GFX11-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
+; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 42
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: s_clause 0xd ; 56-byte Folded Reload
; GFX11-NEXT: scratch_load_b32 v61, off, s33
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
index 06ecf2cc49402..17e3f45a77854 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
@@ -49,8 +49,8 @@ entry:
; CHECK: .name: num_spilled_sgprs
; GFX700: .sgpr_spill_count: 10
; GFX803: .sgpr_spill_count: 10
-; GFX900: .sgpr_spill_count: 62
-; GFX1010: .sgpr_spill_count: 60
+; GFX900: .sgpr_spill_count: 55
+; GFX1010: .sgpr_spill_count: 54
; CHECK: .symbol: num_spilled_sgprs.kd
define amdgpu_kernel void @num_spilled_sgprs(
ptr addrspace(1) %out0, ptr addrspace(1) %out1, [8 x i32],
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index fb0c03543377c..0b5512ef0d821 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -109,25 +109,18 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %bb48
-; CHECK-NEXT: v_readlane_b32 s52, v7, 16
-; CHECK-NEXT: v_readlane_b32 s60, v7, 24
-; CHECK-NEXT: v_readlane_b32 s61, v7, 25
-; CHECK-NEXT: v_readlane_b32 s62, v7, 26
-; CHECK-NEXT: v_readlane_b32 s63, v7, 27
-; CHECK-NEXT: v_readlane_b32 s64, v7, 28
-; CHECK-NEXT: v_readlane_b32 s65, v7, 29
-; CHECK-NEXT: v_readlane_b32 s66, v7, 30
-; CHECK-NEXT: v_readlane_b32 s67, v7, 31
+; CHECK-NEXT: v_readlane_b32 s56, v7, 24
+; CHECK-NEXT: v_readlane_b32 s57, v7, 25
+; CHECK-NEXT: v_readlane_b32 s58, v7, 26
+; CHECK-NEXT: v_readlane_b32 s59, v7, 27
+; CHECK-NEXT: v_readlane_b32 s60, v7, 28
+; CHECK-NEXT: v_readlane_b32 s61, v7, 29
+; CHECK-NEXT: v_readlane_b32 s62, v7, 30
+; CHECK-NEXT: v_readlane_b32 s63, v7, 31
; CHECK-NEXT: v_mov_b32_e32 v1, v2
; CHECK-NEXT: s_and_b64 vcc, exec, -1
-; CHECK-NEXT: v_readlane_b32 s53, v7, 17
-; CHECK-NEXT: v_readlane_b32 s54, v7, 18
-; CHECK-NEXT: v_readlane_b32 s55, v7, 19
-; CHECK-NEXT: image_sample_lz v3, v[2:3], s[60:67], s[68:71] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s56, v7, 20
-; CHECK-NEXT: v_readlane_b32 s57, v7, 21
-; CHECK-NEXT: v_readlane_b32 s58, v7, 22
-; CHECK-NEXT: v_readlane_b32 s59, v7, 23
+; CHECK-NEXT: s_nop 2
+; CHECK-NEXT: image_sample_lz v3, v[2:3], s[56:63], s[68:71] dmask:0x1
; CHECK-NEXT: .LBB0_2: ; %bb50
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
@@ -151,48 +144,34 @@ define void @main(i1 %arg) #0 {
; CHECK-NEXT: ; %bb.5: ; %bb43
; CHECK-NEXT: s_mov_b32 s16, 0
; CHECK-NEXT: s_mov_b32 s17, s16
-; CHECK-NEXT: v_readlane_b32 s44, v7, 16
-; CHECK-NEXT: v_mov_b32_e32 v0, s16
-; CHECK-NEXT: v_mov_b32_e32 v1, s17
+; CHECK-NEXT: v_readlane_b32 s56, v7, 16
+; CHECK-NEXT: v_mov_b32_e32 v2, s16
+; CHECK-NEXT: v_mov_b32_e32 v3, s17
; CHECK-NEXT: s_mov_b32 s18, s16
; CHECK-NEXT: s_mov_b32 s19, s16
-; CHECK-NEXT: v_readlane_b32 s45, v7, 17
-; CHECK-NEXT: v_readlane_b32 s46, v7, 18
-; CHECK-NEXT: v_readlane_b32 s47, v7, 19
-; CHECK-NEXT: v_readlane_b32 s48, v7, 20
-; CHECK-NEXT: v_readlane_b32 s49, v7, 21
-; CHECK-NEXT: v_readlane_b32 s50, v7, 22
-; CHECK-NEXT: v_readlane_b32 s51, v7, 23
-; CHECK-NEXT: v_readlane_b32 s52, v7, 24
-; CHECK-NEXT: v_readlane_b32 s53, v7, 25
-; CHECK-NEXT: v_readlane_b32 s54, v7, 26
-; CHECK-NEXT: v_readlane_b32 s55, v7, 27
-; CHECK-NEXT: v_readlane_b32 s56, v7, 28
-; CHECK-NEXT: v_readlane_b32 s57, v7, 29
-; CHECK-NEXT: v_readlane_b32 s58, v7, 30
-; CHECK-NEXT: v_readlane_b32 s59, v7, 31
-; CHECK-NEXT: image_sample_lz v2, v[0:1], s[44:51], s[16:19] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s44, v7, 0
-; CHECK-NEXT: v_readlane_b32 s52, v7, 8
-; CHECK-NEXT: v_readlane_b32 s53, v7, 9
-; CHECK-NEXT: v_readlane_b32 s54, v7, 10
-; CHECK-NEXT: v_readlane_b32 s55, v7, 11
-; CHECK-NEXT: v_readlane_b32 s56, v7, 12
-; CHECK-NEXT: v_readlane_b32 s57, v7, 13
-; CHECK-NEXT: v_readlane_b32 s58, v7, 14
-; CHECK-NEXT: v_readlane_b32 s59, v7, 15
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, v3
-; CHECK-NEXT: v_readlane_b32 s45, v7, 1
-; CHECK-NEXT: v_readlane_b32 s46, v7, 2
-; CHECK-NEXT: v_readlane_b32 s47, v7, 3
-; CHECK-NEXT: image_sample_lz v0, v[0:1], s[52:59], s[24:27] dmask:0x1
-; CHECK-NEXT: v_readlane_b32 s48, v7, 4
-; CHECK-NEXT: v_readlane_b32 s49, v7, 5
-; CHECK-NEXT: v_readlane_b32 s50, v7, 6
-; CHECK-NEXT: v_readlane_b32 s51, v7, 7
+; CHECK-NEXT: v_readlane_b32 s57, v7, 17
+; CHECK-NEXT: v_readlane_b32 s58, v7, 18
+; CHECK-NEXT: v_readlane_b32 s59, v7, 19
+; CHECK-NEXT: v_readlane_b32 s60, v7, 20
+; CHECK-NEXT: v_readlane_b32 s61, v7, 21
+; CHECK-NEXT: v_readlane_b32 s62, v7, 22
+; CHECK-NEXT: v_readlane_b32 s63, v7, 23
+; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: image_sample_lz v1, v[2:3], s[56:63], s[16:19] dmask:0x1
+; CHECK-NEXT: v_readlane_b32 s56, v7, 8
+; CHECK-NEXT: v_readlane_b32 s57, v7, 9
+; CHECK-NEXT: v_readlane_b32 s58, v7, 10
+; CHECK-NEXT: v_readlane_b32 s59, v7, 11
+; CHECK-NEXT: v_readlane_b32 s60, v7, 12
+; CHECK-NEXT: v_readlane_b32 s61, v7, 13
+; CHECK-NEXT: v_readlane_b32 s62, v7, 14
+; CHECK-NEXT: v_readlane_b32 s63, v7, 15
+; CHECK-NEXT: s_nop 4
+; CHECK-NEXT: image_sample_lz v0, v[2:3], s[56:63], s[24:27] dmask:0x1
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
; CHECK-NEXT: s_waitcnt vmcnt(1)
-; CHECK-NEXT: buffer_store_dwordx3 v[2:4], off, s[16:19], 0
+; CHECK-NEXT: buffer_store_dwordx3 v[1:3], off, s[16:19], 0
; CHECK-NEXT: s_waitcnt vmcnt(1)
; CHECK-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; CHECK-NEXT: ; implicit-def: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
index 3ed98899d6019..c30d98dc2e3ba 100644
--- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir
@@ -1,3 +1,4 @@
+# XFAIL: *
# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -simplify-mir -start-before=greedy,2 -stress-regalloc=4 -stop-before=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s
# RUN: FileCheck -check-prefix=ERR %s < %t.err
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
index d455dae4a46c7..d01818d79b4cb 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill-inspect-subrange.mir
@@ -58,7 +58,8 @@ body: |
; CHECK-NEXT: SI_SPILL_S512_SAVE renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s512) into %stack.0, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_2:%[0-9]+]].sub0:vreg_96 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = SI_SPILL_S512_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s512) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S256_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = lr-split COPY killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
; CHECK-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = IMPLICIT_DEF
; CHECK-NEXT: dead undef [[IMAGE_SAMPLE_LZ_V1_V2_3:%[0-9]+]].sub0:vreg_128 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF2]], undef renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: S_BRANCH %bb.2
@@ -82,7 +83,7 @@ body: |
; CHECK-NEXT: bb.7:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.7, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.6
diff --git a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
index a8419a4bfc749..06cd1396f24c7 100644
--- a/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/infloop-subrange-spill.mir
@@ -79,7 +79,7 @@ body: |
; CHECK-NEXT: bb.6:
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19:0x000000000000FFFF, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x00000000FFFFFFFF
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr8_sgpr9_sgpr10_sgpr11, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
+ ; CHECK-NEXT: dead [[IMAGE_SAMPLE_LZ_V1_V2_5:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_LZ_V1_V2 undef [[DEF]], renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, undef renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
; CHECK-NEXT: renamable $sgpr25 = COPY undef renamable $sgpr24, implicit-def $sgpr24
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.6, implicit undef $vcc
; CHECK-NEXT: S_BRANCH %bb.5
diff --git a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
index 6f76da9017343..6d2baa3fce29a 100644
--- a/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
+++ b/llvm/test/CodeGen/AMDGPU/ra-inserted-scalar-instructions.mir
@@ -226,8 +226,8 @@ body: |
; GCN-NEXT: dead [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
; GCN-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GCN-NEXT: [[S_LOAD_DWORD_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_MOV_B64_1]], 0, 0 :: ("amdgpu-noclobber" load (s32), addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE1:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = lr-split COPY [[SI_SPILL_S64_RESTORE1]].sub1
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.2, 1, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; GCN-NEXT: undef [[COPY2:%[0-9]+]].sub1:sgpr_64 = lr-split COPY [[SI_SPILL_S32_RESTORE]]
; GCN-NEXT: [[COPY2:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 1
; GCN-NEXT: S_CBRANCH_SCC1 %bb.10, implicit undef $scc
; GCN-NEXT: S_BRANCH %bb.7
@@ -259,13 +259,13 @@ body: |
; GCN-NEXT: successors: %bb.11(0x40000000), %bb.12(0x40000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORD_IMM2]], 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.17, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.17, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_]], [[SI_SPILL_S32_RESTORE]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE1:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.15, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.15, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE1]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE1:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.17, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.17, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_]], [[SI_SPILL_S32_RESTORE1]], implicit-def dead $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE2:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.15, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.15, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE2]], 0, implicit-def $scc
; GCN-NEXT: dead [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE2:%[0-9]+]]:sreg_64_xexec = SI_SPILL_S64_RESTORE %stack.18, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.18, align 4, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S64_RESTORE2]].sub1, 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE3:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.18, 1, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.18, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE3]], 0, implicit-def $scc
; GCN-NEXT: dead [[DEF5:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
; GCN-NEXT: [[SI_SPILL_S256_RESTORE:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.20, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.20, align 4, addrspace 5)
; GCN-NEXT: undef [[COPY3:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE]].sub0 {
@@ -284,8 +284,8 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_7:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub4, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_8:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_9:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[S_LOAD_DWORDX8_IMM]].sub6, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S128_RESTORE:%[0-9]+]]:sgpr_128 = SI_SPILL_S128_RESTORE %stack.14, 0, implicit $exec, implicit $sgpr32 :: (load (s128) from %stack.14, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = lr-split COPY [[SI_SPILL_S128_RESTORE]].sub0_sub1_sub2
+ ; GCN-NEXT: [[SI_SPILL_S96_RESTORE:%[0-9]+]]:sgpr_96 = SI_SPILL_S96_RESTORE %stack.14, 0, implicit $exec, implicit $sgpr32 :: (load (s96) from %stack.14, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1_sub2:sgpr_128 = lr-split COPY [[SI_SPILL_S96_RESTORE]]
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_10:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_11:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub1, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_12:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY4]].sub2, 0, implicit $mode, implicit $exec
@@ -307,23 +307,23 @@ body: |
; GCN-NEXT: dead [[S_AND_B32_3:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[V_CMP_GT_F32_e64_8]], undef [[V_CMP_GT_F32_e64_9]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_14:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY5]].sub2, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_OR_B32_2:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY5]].sub5, [[COPY5]].sub7, implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE2:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.10, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.10, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: [[SI_SPILL_S224_RESTORE:%[0-9]+]]:sgpr_224 = SI_SPILL_S224_RESTORE %stack.10, 0, implicit $exec, implicit $sgpr32 :: (load (s224) from %stack.10, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY6:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S224_RESTORE]]
; GCN-NEXT: dead [[S_OR_B32_3:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub0, [[COPY6]].sub1, implicit-def dead $scc
; GCN-NEXT: dead [[S_OR_B32_4:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY6]].sub2, undef [[S_OR_B32_3]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE2:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.9, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.9, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_4:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_3]], [[SI_SPILL_S32_RESTORE2]], implicit-def dead $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE4:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.9, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.9, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_4:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_3]], [[SI_SPILL_S32_RESTORE4]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_15:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub3, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_16:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub4, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_17:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_18:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY6]].sub6, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE3:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.11, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.11, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_19:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE3]], 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE3:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.13, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.13, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub0 {
- ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub2
- ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub4
- ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE3]].sub7
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE5:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.11, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.11, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_19:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE5]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S256_RESTORE2:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.13, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.13, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY7:%[0-9]+]].sub0:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].sub0 {
+ ; GCN-NEXT: internal [[COPY7]].sub2:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].sub2
+ ; GCN-NEXT: internal [[COPY7]].sub4:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].sub4
+ ; GCN-NEXT: internal [[COPY7]].sub7:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE2]].sub7
; GCN-NEXT: }
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_20:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_21:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY7]].sub2, 0, implicit $mode, implicit $exec
@@ -367,8 +367,8 @@ body: |
; GCN-NEXT: dead [[DEF15:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[S_AND_B32_7:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF15]], undef [[DEF14]], implicit-def dead $scc
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_33:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY9]].sub12, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE4:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.6, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.6, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE4]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16
+ ; GCN-NEXT: [[SI_SPILL_S224_RESTORE1:%[0-9]+]]:sgpr_224 = SI_SPILL_S224_RESTORE %stack.6, 0, implicit $exec, implicit $sgpr32 :: (load (s224) from %stack.6, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY10:%[0-9]+]].lo16_hi16_sub1_lo16_sub1_hi16_sub2_lo16_sub2_hi16_sub3_lo16_sub3_hi16_sub4_lo16_sub4_hi16_sub5_lo16_sub5_hi16_sub6_lo16_sub6_hi16:sgpr_256 = lr-split COPY [[SI_SPILL_S224_RESTORE1]]
; GCN-NEXT: dead [[S_OR_B32_6:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY10]].sub0, [[COPY9]].sub15, implicit-def dead $scc
; GCN-NEXT: dead [[DEF16:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_34:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub1, 0, implicit $mode, implicit $exec
@@ -380,23 +380,23 @@ body: |
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_38:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub5, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_39:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY10]].sub6, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_AND_B32_8:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[DEF18]], undef [[DEF17]], implicit-def dead $scc
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE5:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.4, align 4, addrspace 5)
- ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = lr-split COPY [[SI_SPILL_S256_RESTORE5]].sub0_sub1_sub2_sub3_sub4_sub5
+ ; GCN-NEXT: [[SI_SPILL_S192_RESTORE:%[0-9]+]]:sgpr_192 = SI_SPILL_S192_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s192) from %stack.4, align 4, addrspace 5)
+ ; GCN-NEXT: undef [[COPY11:%[0-9]+]].sub0_sub1_sub2_sub3_sub4_sub5:sgpr_256 = lr-split COPY [[SI_SPILL_S192_RESTORE]]
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_40:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub0, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_41:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub1, 0, implicit $mode, implicit $exec
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE4:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.3, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_42:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE4]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE6:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.3, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_42:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE6]], 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_43:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub2, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[V_CMP_GT_F32_e64_44:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[COPY11]].sub3, 0, implicit $mode, implicit $exec
; GCN-NEXT: dead [[S_OR_B32_7:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY11]].sub4, [[COPY11]].sub5, implicit-def dead $scc
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE4]], 0, implicit-def $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE5:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.5, addrspace 5)
- ; GCN-NEXT: dead [[S_AND_B32_9:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_7]], [[SI_SPILL_S32_RESTORE5]], implicit-def dead $scc
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE6]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE7:%[0-9]+]]:sreg_32_xm0_xexec = SI_SPILL_S32_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.5, addrspace 5)
+ ; GCN-NEXT: dead [[S_AND_B32_9:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_OR_B32_7]], [[SI_SPILL_S32_RESTORE7]], implicit-def dead $scc
; GCN-NEXT: dead [[S_LOAD_DWORD_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY2]], 0, 0 :: ("amdgpu-noclobber" load (s32), align 8, addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S256_RESTORE6:%[0-9]+]]:sgpr_256 = SI_SPILL_S256_RESTORE %stack.8, 0, implicit $exec, implicit $sgpr32 :: (load (s256) from %stack.8, align 4, addrspace 5)
- ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S256_RESTORE6]].sub7, 0, implicit-def $scc
- ; GCN-NEXT: [[SI_SPILL_S32_RESTORE6:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.7, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.7, addrspace 5)
- ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_45:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE6]], 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE8:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.8, 7, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.8, addrspace 5)
+ ; GCN-NEXT: S_CMP_EQ_U32 [[SI_SPILL_S32_RESTORE8]], 0, implicit-def $scc
+ ; GCN-NEXT: [[SI_SPILL_S32_RESTORE9:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.7, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.7, addrspace 5)
+ ; GCN-NEXT: dead [[V_CMP_GT_F32_e64_45:%[0-9]+]]:sreg_32 = V_CMP_GT_F32_e64 0, 0, 0, [[SI_SPILL_S32_RESTORE9]], 0, implicit $mode, implicit $exec
; GCN-NEXT: [[DEF19:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GCN-NEXT: dead [[S_AND_B32_10:%[0-9]+]]:sreg_32 = S_AND_B32 [[DEF19]], undef [[S_LOAD_DWORD_IMM6]], implicit-def dead $scc
; GCN-NEXT: dead [[S_AND_B32_11:%[0-9]+]]:sreg_32 = S_AND_B32 undef [[S_AND_B32_10]], [[S_MOV_B32_2]], implicit-def dead $scc
@@ -408,10 +408,10 @@ body: |
; GCN-NEXT: successors: %bb.12(0x80000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.12:
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE3:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.1, align 4, addrspace 5)
- ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE3]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
- ; GCN-NEXT: [[SI_SPILL_S64_RESTORE4:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE4]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GCN-NEXT: [[SI_SPILL_S64_RESTORE1:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.1, align 4, addrspace 5)
+ ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; GCN-NEXT: [[SI_SPILL_S64_RESTORE2:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; GCN-NEXT: GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], [[SI_SPILL_S64_RESTORE2]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
; GCN-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1, %bb.2
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 327e6dc5b2519..debe44bfe215f 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -43,102 +43,86 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr15, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.0, addrspace 5)
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr14, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.1, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.3, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.4, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr36_sgpr37 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr38_sgpr39 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr48_sgpr49 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr56 = S_MOV_B32 0
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.5, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.5, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr50_sgpr51 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit $exec
- ; CHECK-NEXT: renamable $sgpr100_sgpr101 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr52_sgpr53 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr57 = S_MOV_B32 1083786240
; CHECK-NEXT: SI_SPILL_S1024_SAVE renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, %stack.2, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.2, align 4, addrspace 5)
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr100_sgpr101, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr52_sgpr53, implicit-def dead $scc
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_1024_align2 = COPY [[COPY]]
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
; CHECK-NEXT: S_BRANCH %bb.17
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr41 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr42 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr43 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr44 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr45 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr46 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr47 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr48 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr49 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr50 = COPY renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr51 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52 = lr-split COPY renamable $sgpr56
- ; CHECK-NEXT: renamable $sgpr53 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr56_sgpr57 = lr-split COPY renamable $sgpr52_sgpr53
- ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53 = lr-split COPY renamable $sgpr56_sgpr57
- ; CHECK-NEXT: renamable $sgpr54 = COPY killed renamable $sgpr76
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
- ; CHECK-NEXT: renamable $sgpr48_sgpr49_sgpr50 = lr-split COPY renamable $sgpr52_sgpr53_sgpr54
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54 = lr-split COPY renamable $sgpr48_sgpr49_sgpr50
- ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47
- ; CHECK-NEXT: renamable $sgpr55 = COPY killed renamable $sgpr68
- ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr56 = COPY killed renamable $sgpr72
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr57 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr58 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr59 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr60 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr61 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr62 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr63 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr64 = COPY killed renamable $sgpr80
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr65 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr66 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr68 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = lr-split COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
- ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = lr-split COPY renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
- ; CHECK-NEXT: renamable $sgpr64 = lr-split COPY renamable $sgpr68
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr65 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr66 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr84
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit $exec
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr68_sgpr69 = lr-split COPY killed renamable $sgpr12_sgpr13
+ ; CHECK-NEXT: renamable $sgpr56 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr57 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr58 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr59 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr60 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr61 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr62 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr63 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr64 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr65 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr66 = COPY renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr67 = COPY killed renamable $sgpr68
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr68 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr69 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr70 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr71 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr72 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr73 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr74 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr75 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr76 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr77 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr78 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr79 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr80 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr81 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr82 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr83 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr84 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr85 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr86 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: renamable $sgpr9 = SI_SPILL_S32_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr87 = COPY killed renamable $sgpr9
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87, implicit $exec
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.11, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
@@ -165,23 +149,22 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.12(0x40000000), %bb.6(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.4, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr12_sgpr13, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = S_AND_B64 renamable $sgpr38_sgpr39, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
; CHECK-NEXT: renamable $sgpr54_sgpr55 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: $exec = S_MOV_B64_term killed renamable $sgpr12_sgpr13
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.12, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
; CHECK-NEXT: successors: %bb.7(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
+ ; CHECK-NEXT: dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr48_sgpr49, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7:
; CHECK-NEXT: successors: %bb.8(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $sgpr64_sgpr65 = nofpexcept V_CMP_NLT_F64_e64 0, undef $sgpr4_sgpr5, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
; CHECK-NEXT: renamable $sgpr66_sgpr67 = nofpexcept V_CMP_NLT_F64_e64 0, 4607182418800017408, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
@@ -189,19 +172,19 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.8:
; CHECK-NEXT: successors: %bb.10(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr64_sgpr65, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.10, implicit $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.9:
; CHECK-NEXT: successors: %bb.10(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.2, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY killed renamable $sgpr84_sgpr85, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.2, 16, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY killed renamable $sgpr12_sgpr13, implicit $exec
; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR undef %18:vgpr_32, [[COPY2]], undef renamable $sgpr4_sgpr5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
- ; CHECK-NEXT: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr14_sgpr15, implicit $exec
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
; CHECK-NEXT: dead renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_1]], implicit $exec
; CHECK-NEXT: renamable $sgpr82 = S_ADD_U32 renamable $sgpr8, 32, implicit-def dead $scc
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
@@ -215,13 +198,11 @@ body: |
; CHECK-NEXT: $sgpr13 = SI_SPILL_S32_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.0, addrspace 5)
; CHECK-NEXT: renamable $sgpr84 = lr-split COPY killed renamable $sgpr8
; CHECK-NEXT: renamable $sgpr33 = lr-split COPY killed renamable $sgpr16
- ; CHECK-NEXT: renamable $sgpr48_sgpr49 = lr-split COPY killed renamable $sgpr14_sgpr15
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: $sgpr8_sgpr9 = COPY renamable $sgpr82_sgpr83
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr8_sgpr9
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = lr-split COPY killed renamable $sgpr48_sgpr49
; CHECK-NEXT: renamable $sgpr16 = lr-split COPY killed renamable $sgpr33
; CHECK-NEXT: renamable $sgpr4_sgpr5 = lr-split COPY killed renamable $sgpr68_sgpr69
; CHECK-NEXT: renamable $sgpr6_sgpr7 = lr-split COPY killed renamable $sgpr70_sgpr71
@@ -234,44 +215,42 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.10:
; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.12(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.8, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.12
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.11:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.17(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.1, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.17
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.12:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.13(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_MOV_B64_term killed renamable $sgpr54_sgpr55
; CHECK-NEXT: S_CBRANCH_EXECZ %bb.11, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.13:
; CHECK-NEXT: successors: %bb.15(0x40000000), %bb.14(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.5, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr50_sgpr51, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.15, implicit $vcc
; CHECK-NEXT: S_BRANCH %bb.14
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.14:
; CHECK-NEXT: successors: %bb.15(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.15:
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.16(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
+ ; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.3, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, renamable $sgpr36_sgpr37, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.11, implicit $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.16:
diff --git a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
index a0993d7e35176..14fb667e956fb 100644
--- a/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
+++ b/llvm/test/CodeGen/AMDGPU/regpressure-mitigation-with-subreg-reload.mir
@@ -1,13 +1,13 @@
-# RUN: not llc -mcpu=gfx1100 -mtriple=amdgcn-amd-amdhsa -stress-regalloc=4 -run-pass=greedy -filetype=null %s 2>&1 | FileCheck %s
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mcpu=gfx1100 -mtriple=amdgcn-amd-amdhsa -stress-regalloc=4 --verify-machineinstrs -run-pass=greedy -o - %s | FileCheck %s
# This test demonstrates register pressure mitigation through subreg-aware
-# reloads during register allocation. Currently, the test would fail during
-# RA due to insufficient registers. Subreg-aware reload, when implemented,
-# mitigates register pressure by reloading only the used portions of tuple
-# registers from their spilled locations instead of reloading entire tuples,
-# thereby enhancing register allocation.
-
-# CHECK: error: <unknown>:0:0: ran out of registers during register allocation
+# reloads during register allocation. This test originally failed with
+# insufficient registers during RA due to a high register pressure. The
+# subreg-aware reload, when implemented, mitigated register pressure by
+# reloading only the used portions of tuple registers from their spilled
+# locations instead of reloading entire tuples, thereby enhancing register
+# allocation.
---
name: subreg-reload
@@ -19,6 +19,40 @@ body: |
bb.0:
liveins: $vgpr1, $sgpr0_sgpr1, $sgpr2_sgpr3
+ ; CHECK-LABEL: name: subreg-reload
+ ; CHECK: liveins: $vgpr1, $sgpr0_sgpr1, $sgpr2_sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY renamable $sgpr0_sgpr1
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE [[COPY1]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY1]], 0, 0, implicit $exec, implicit $flat_scr
+ ; CHECK-NEXT: SI_SPILL_V128_SAVE [[FLAT_LOAD_DWORDX4_]], %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: undef early-clobber %31.sub0_sub1:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE]], 42, 0, 0, implicit $exec
+ ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:vreg_64 = COPY %31.sub1
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE1:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 4, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %31.sub1_sub2:vreg_128, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE1]], 42, [[COPY2]], 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V128_SAVE %31, %stack.3, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE2:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %24:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE2]], 42, 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE %24, %stack.2, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE3:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.2, $sgpr32, 4, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:vreg_64 = COPY [[SI_SPILL_V32_RESTORE3]]
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE4:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.1, $sgpr32, 12, implicit $exec :: (load (s32) from %stack.1, addrspace 5)
+ ; CHECK-NEXT: early-clobber %35:vreg_64, $sgpr_null = V_MAD_U64_U32_gfx11_e64 [[SI_SPILL_V32_RESTORE4]], 42, [[COPY3]], 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE %35, %stack.4, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.4, align 4, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE5:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY4:%[0-9]+]].sub0_sub1:vreg_128 = lr-split COPY [[SI_SPILL_V64_RESTORE]]
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]].sub2:vreg_128 = COPY [[SI_SPILL_V32_RESTORE5]]
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE6:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.4, addrspace 5)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]].sub3:vreg_128 = COPY [[SI_SPILL_V32_RESTORE6]]
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128 = lr-split COPY [[COPY4]]
+ ; CHECK-NEXT: $vgpr31 = COPY [[COPY]]
+ ; CHECK-NEXT: INLINEASM &"; use v1", sideeffect attdialect, regdef:VS_16_Lo128, $vgpr1
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE1:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: FLAT_STORE_DWORDX4 [[SI_SPILL_V64_RESTORE1]], [[COPY5]], 0, 0, implicit $exec, implicit $flat_scr
+ ; CHECK-NEXT: SI_RETURN
%1:vgpr_32 = COPY $vgpr1
%2:vreg_64 = COPY killed renamable $sgpr0_sgpr1
%3:vreg_128 = FLAT_LOAD_DWORDX4 %2, 0, 0, implicit $exec, implicit $flat_scr
diff --git a/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir b/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
new file mode 100644
index 0000000000000..f363f17e16ae1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/skip-partial-reload-for-16bit-regaccess.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -stress-regalloc=3 -run-pass=greedy -verify-machineinstrs %s -o - | FileCheck %s
+
+# Do not insert partial reloads for lo16 and hi16 accesses. They must remain full reloads of their
+# original register (be it a 32-bit or any larger tuple) before each use. We need to specifiy the
+# 32-bit stack offset in the spill/reload instructions and the register allocator currently cannot
+# reliably assign the expected 16-bit slices (lo16 or hi16) of a physical register if we introduce
+# 16-bit reloads.
+
+# This test ensures the spill reloads inserted during RA should always reload the full register
+# before their 16-bit register uses.
+
+---
+name: reload_v32_for_lo16_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+ frameOffsetReg: $sgpr33
+ stackPtrOffsetReg: $sgpr32
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr0
+
+ ; CHECK-LABEL: name: reload_v32_for_lo16_use
+ ; CHECK: liveins: $vgpr0, $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_UBYTE_D16_t16_:%[0-9]+]].lo16:vgpr_32 = GLOBAL_LOAD_UBYTE_D16_t16 [[V_MOV_B]], 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V32_SAVE [[GLOBAL_LOAD_UBYTE_D16_t16_]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5)
+ ; CHECK-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1023, $vgpr0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 1, [[V_AND_B32_e64_]], implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B16_t16 [[V_LSHLREV_B32_e64_]], [[V_MOV_B16_t16_e64_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: dead [[DS_READ_U16_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U16_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[SI_SPILL_V32_RESTORE:%[0-9]+]]:vgpr_32 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5)
+ ; CHECK-NEXT: DS_WRITE_B8_t16 [[V_MOV_B32_e32_]], [[SI_SPILL_V32_RESTORE]].lo16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %1:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ undef %3.lo16:vgpr_32 = GLOBAL_LOAD_UBYTE_D16_t16 %1:vreg_64, 0, 0, implicit $exec
+ %2:vgpr_32 = V_AND_B32_e64 1023, $vgpr0, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32_gfx9 %4:vgpr_32, %4:vgpr_32, 0, 0, implicit $exec
+ %5:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ %6:vgpr_32 = V_LSHLREV_B32_e64 1, %2:vgpr_32, implicit $exec
+ DS_WRITE_B16_t16 %6:vgpr_32, %5:vgpr_16, 0, 0, implicit $exec
+ %7:vgpr_32 = DS_READ_U16_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ DS_WRITE_B8_t16 %4:vgpr_32, %3.lo16:vgpr_32, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: reload_v64_for_sub1_hi16_use
+tracksRegLiveness: true
+machineFunctionInfo:
+ scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+ frameOffsetReg: $sgpr33
+ stackPtrOffsetReg: $sgpr32
+body: |
+ bb.0:
+ liveins: $sgpr0
+
+ ; CHECK-LABEL: name: reload_v64_for_sub1_hi16_use
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_128 = IMPLICIT_DEF
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_UBYTE_D16_t16_:%[0-9]+]].sub1_hi16:vreg_64 = GLOBAL_LOAD_UBYTE_D16_t16 [[V_MOV_B]], 0, 0, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_V64_SAVE [[GLOBAL_LOAD_UBYTE_D16_t16_]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 1, [[DEF]].sub1, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B16_t16 [[V_LSHLREV_B32_e64_]], [[V_MOV_B16_t16_e64_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: dead [[DS_READ_U16_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_U16_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+ ; CHECK-NEXT: [[SI_SPILL_V64_RESTORE:%[0-9]+]]:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: DS_WRITE_B8_t16 [[V_MOV_B32_e32_]], [[SI_SPILL_V64_RESTORE]].sub1_hi16, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %1:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ %2:vreg_128 = IMPLICIT_DEF
+ undef %3.sub1_hi16:vreg_64 = GLOBAL_LOAD_UBYTE_D16_t16 %1:vreg_64, 0, 0, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ DS_WRITE_B32_gfx9 %4:vgpr_32, %4:vgpr_32, 0, 0, implicit $exec
+ %5:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ %6:vgpr_32 = V_LSHLREV_B32_e64 1, %2.sub1:vreg_128, implicit $exec
+ DS_WRITE_B16_t16 %6:vgpr_32, %5:vgpr_16, 0, 0, implicit $exec
+ %7:vgpr_32 = DS_READ_U16_gfx9 %4:vgpr_32, 0, 0, implicit $exec
+ DS_WRITE_B8_t16 %4:vgpr_32, %3.sub1_hi16:vreg_64, 0, 0, implicit $exec
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
index 5c829d7ec88ef..fcf3547929268 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-scavenge-offset.ll
@@ -10126,41 +10126,35 @@ define amdgpu_kernel void @test_limited_sgpr(ptr addrspace(1) %out, ptr addrspac
; GFX6-NEXT: ;;#ASMEND
; GFX6-NEXT: .LBB1_2: ; %ret
; GFX6-NEXT: s_or_b64 exec, exec, s[34:35]
-; GFX6-NEXT: s_mov_b64 s[4:5], exec
-; GFX6-NEXT: s_mov_b64 exec, 15
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_mov_b64 exec, 3
; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT: s_mov_b32 s6, 0x80400
+; GFX6-NEXT: s_mov_b32 s2, 0x80400
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s6 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s2 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readlane_b32 s0, v4, 0
-; GFX6-NEXT: v_readlane_b32 s1, v4, 1
-; GFX6-NEXT: v_readlane_b32 s2, v4, 2
-; GFX6-NEXT: v_readlane_b32 s3, v4, 3
+; GFX6-NEXT: v_readlane_b32 s36, v4, 0
+; GFX6-NEXT: v_readlane_b32 s37, v4, 1
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: s_mov_b64 exec, s[4:5]
-; GFX6-NEXT: s_mov_b64 s[36:37], s[0:1]
-; GFX6-NEXT: s_mov_b64 s[4:5], exec
-; GFX6-NEXT: s_mov_b64 exec, 15
+; GFX6-NEXT: s_mov_b64 exec, s[0:1]
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_mov_b64 exec, 3
; GFX6-NEXT: buffer_store_dword v4, off, s[40:43], 0
-; GFX6-NEXT: s_mov_b32 s6, 0x80800
+; GFX6-NEXT: s_mov_b32 s2, 0x80800
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s6 ; 4-byte Folded Reload
+; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], s2 ; 4-byte Folded Reload
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readlane_b32 s0, v4, 0
-; GFX6-NEXT: v_readlane_b32 s1, v4, 1
-; GFX6-NEXT: v_readlane_b32 s2, v4, 2
-; GFX6-NEXT: v_readlane_b32 s3, v4, 3
+; GFX6-NEXT: v_readlane_b32 s38, v4, 2
+; GFX6-NEXT: v_readlane_b32 s39, v4, 3
; GFX6-NEXT: buffer_load_dword v4, off, s[40:43], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: s_mov_b64 exec, s[4:5]
+; GFX6-NEXT: s_mov_b64 exec, s[0:1]
; GFX6-NEXT: s_mov_b32 s0, 0x86800
; GFX6-NEXT: buffer_load_dword v7, off, s[40:43], s0 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v8, off, s[40:43], s0 offset:4 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v9, off, s[40:43], s0 offset:8 ; 4-byte Folded Reload
; GFX6-NEXT: buffer_load_dword v10, off, s[40:43], s0 offset:12 ; 4-byte Folded Reload
-; GFX6-NEXT: s_mov_b64 s[38:39], s[2:3]
; GFX6-NEXT: s_mov_b32 s0, 0x86400
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_store_dwordx4 v[7:10], v[5:6], s[36:39], 0 addr64 offset:240
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
index 530d868ba3085..21ce4dfc4c7a9 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
@@ -24,8 +24,8 @@ body: |
; RA-NEXT: bb.1:
; RA-NEXT: successors: %bb.2(0x80000000)
; RA-NEXT: {{ $}}
- ; RA-NEXT: [[SI_SPILL_S1024_RESTORE:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = lr-split COPY [[SI_SPILL_S1024_RESTORE]].sub0_sub1
+ ; RA-NEXT: [[SI_SPILL_S64_RESTORE:%[0-9]+]]:sgpr_64 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; RA-NEXT: undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = lr-split COPY [[SI_SPILL_S64_RESTORE]]
; RA-NEXT: [[COPY:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY]].sub0
; RA-NEXT: [[COPY:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: [[COPY:%[0-9]+]].sub4:sgpr_1024 = COPY [[COPY]].sub0
@@ -55,8 +55,8 @@ body: |
; RA-NEXT: [[COPY:%[0-9]+]].sub28:sgpr_1024 = COPY [[COPY]].sub0
; RA-NEXT: [[COPY:%[0-9]+]].sub29:sgpr_1024 = COPY [[COPY]].sub1
; RA-NEXT: SI_SPILL_S1024_SAVE [[COPY]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
- ; RA-NEXT: [[SI_SPILL_S1024_RESTORE1:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
- ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = lr-split COPY [[SI_SPILL_S1024_RESTORE1]].sub0
+ ; RA-NEXT: [[SI_SPILL_S32_RESTORE:%[0-9]+]]:sgpr_32 = SI_SPILL_S32_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.1, addrspace 5)
+ ; RA-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = lr-split COPY [[SI_SPILL_S32_RESTORE]]
; RA-NEXT: [[COPY1:%[0-9]+]].sub1:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY1]].sub0
; RA-NEXT: [[COPY1:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY1]].sub0
@@ -113,7 +113,8 @@ body: |
; VR-NEXT: successors: %bb.2(0x80000000)
; VR-NEXT: liveins: $sgpr30_sgpr31, $sgpr34_sgpr35
; VR-NEXT: {{ $}}
- ; VR-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr4_sgpr5 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.0, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr36_sgpr37 = lr-split COPY killed renamable $sgpr4_sgpr5
; VR-NEXT: renamable $sgpr38 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr39 = COPY renamable $sgpr37
; VR-NEXT: renamable $sgpr40 = COPY renamable $sgpr36
@@ -143,7 +144,8 @@ body: |
; VR-NEXT: renamable $sgpr64 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr65 = COPY renamable $sgpr37
; VR-NEXT: SI_SPILL_S1024_SAVE killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
- ; VR-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+ ; VR-NEXT: renamable $sgpr4 = SI_SPILL_S32_RESTORE %stack.1, 0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.1, addrspace 5)
+ ; VR-NEXT: renamable $sgpr36 = lr-split COPY killed renamable $sgpr4
; VR-NEXT: renamable $sgpr37 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr38 = COPY renamable $sgpr36
; VR-NEXT: renamable $sgpr39 = COPY renamable $sgpr36
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
index 0bfd490a0ae68..f10071b96e23d 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-live-lanes.mir
@@ -312,9 +312,9 @@ body: |
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY67:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY67]], [[S_MOV_B32_]], 0, 224, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.10, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.10, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE]].sub0 {
- ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.10, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.10, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE]].sub0 {
+ ; CHECK-NEXT: internal [[COPY68]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY68]].sub0 {
; CHECK-NEXT: internal [[COPY69]].sub2:vreg_128 = lr-split COPY [[COPY68]].sub2
@@ -331,9 +331,9 @@ body: |
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY71:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY71]], [[S_MOV_B32_]], 0, 192, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE1:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.9, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.9, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE1]].sub0 {
- ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE1]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE1:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.9, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.9, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE1]].sub0 {
+ ; CHECK-NEXT: internal [[COPY72]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE1]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY72]].sub0 {
; CHECK-NEXT: internal [[COPY73]].sub2:vreg_128 = lr-split COPY [[COPY72]].sub2
@@ -341,9 +341,9 @@ body: |
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY73:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY73]], [[S_MOV_B32_]], 0, 208, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE2:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.11, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.11, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE2]].sub0 {
- ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE2]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE2:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.11, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.11, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE2]].sub0 {
+ ; CHECK-NEXT: internal [[COPY74]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE2]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY74]].sub0 {
; CHECK-NEXT: internal [[COPY75]].sub2:vreg_128 = lr-split COPY [[COPY74]].sub2
@@ -351,9 +351,9 @@ body: |
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY75:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY75]], [[S_MOV_B32_]], 0, 160, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE3:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.8, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.8, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE3]].sub0 {
- ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE3]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE3:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.8, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.8, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE3]].sub0 {
+ ; CHECK-NEXT: internal [[COPY76]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE3]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY76]].sub0 {
; CHECK-NEXT: internal [[COPY77]].sub2:vreg_128 = lr-split COPY [[COPY76]].sub2
@@ -379,9 +379,9 @@ body: |
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY81:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY81]], [[S_MOV_B32_]], 0, 144, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE4:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.6, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE4]].sub0 {
- ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE4]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE4:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.6, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE4]].sub0 {
+ ; CHECK-NEXT: internal [[COPY82]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE4]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY82]].sub0 {
; CHECK-NEXT: internal [[COPY83]].sub2:vreg_128 = lr-split COPY [[COPY82]].sub2
@@ -389,9 +389,9 @@ body: |
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY83:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY83]], [[S_MOV_B32_]], 0, 96, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE5:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.7, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.7, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE5]].sub0 {
- ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE5]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE5:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.7, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.7, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE5]].sub0 {
+ ; CHECK-NEXT: internal [[COPY84]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE5]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY84]].sub0 {
; CHECK-NEXT: internal [[COPY85]].sub2:vreg_128 = lr-split COPY [[COPY84]].sub2
@@ -399,9 +399,9 @@ body: |
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY85:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY85]], [[S_MOV_B32_]], 0, 112, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE6:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.5, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE6]].sub0 {
- ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE6]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE6:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.5, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE6]].sub0 {
+ ; CHECK-NEXT: internal [[COPY86]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE6]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY86]].sub0 {
; CHECK-NEXT: internal [[COPY87]].sub2:vreg_128 = lr-split COPY [[COPY86]].sub2
@@ -409,9 +409,9 @@ body: |
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY87:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY87]], [[S_MOV_B32_]], 0, 64, 0, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE7:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE7]].sub0 {
- ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE7]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE7:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.4, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE7]].sub0 {
+ ; CHECK-NEXT: internal [[COPY88]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE7]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY88]].sub0 {
; CHECK-NEXT: internal [[COPY89]].sub2:vreg_128 = lr-split COPY [[COPY88]].sub2
@@ -419,9 +419,9 @@ body: |
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY89:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY89]], [[S_MOV_B32_]], 0, 80, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE8:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.3, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE8]].sub0 {
- ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE8]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE8:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE8]].sub0 {
+ ; CHECK-NEXT: internal [[COPY90]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE8]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY90]].sub0 {
; CHECK-NEXT: internal [[COPY91]].sub2:vreg_128 = lr-split COPY [[COPY90]].sub2
@@ -429,9 +429,9 @@ body: |
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY91:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY91]], [[S_MOV_B32_]], 0, 32, 0, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE9:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE9]].sub0 {
- ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE9]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE9:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE9]].sub0 {
+ ; CHECK-NEXT: internal [[COPY92]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE9]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY92]].sub0 {
; CHECK-NEXT: internal [[COPY93]].sub2:vreg_128 = lr-split COPY [[COPY92]].sub2
@@ -439,9 +439,9 @@ body: |
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY93:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY93]], [[S_MOV_B32_]], 0, 48, 0, 0, implicit $exec :: (store (s128), addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE10:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE10]].sub0 {
- ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE10]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE10:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.1, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE10]].sub0 {
+ ; CHECK-NEXT: internal [[COPY94]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE10]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY94]].sub0 {
; CHECK-NEXT: internal [[COPY95]].sub2:vreg_128 = lr-split COPY [[COPY94]].sub2
@@ -449,9 +449,9 @@ body: |
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub1:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: [[COPY95:%[0-9]+]].sub3:vreg_128 = COPY [[V_LSHRREV_B32_e32_26]].sub1
; CHECK-NEXT: BUFFER_STORE_DWORDX4_OFFSET [[COPY95]], [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (store (s128), align 512, addrspace 1)
- ; CHECK-NEXT: [[SI_SPILL_V128_RESTORE11:%[0-9]+]]:vreg_128 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE11]].sub0 {
- ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V128_RESTORE11]].sub2
+ ; CHECK-NEXT: [[SI_SPILL_V96_RESTORE11:%[0-9]+]]:vreg_96 = SI_SPILL_V96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5)
+ ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE11]].sub0 {
+ ; CHECK-NEXT: internal [[COPY96]].sub2:vreg_128 = lr-split COPY [[SI_SPILL_V96_RESTORE11]].sub2
; CHECK-NEXT: }
; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_128 = lr-split COPY [[COPY96]].sub0 {
; CHECK-NEXT: internal [[COPY97]].sub2:vreg_128 = lr-split COPY [[COPY96]].sub2
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
index c40e35bbe1754..45d7d16da6562 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
@@ -759,16 +759,16 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_cbranch_execz .LBB0_41
; CHECK-NEXT: ; %bb.8: ; %bb0b
; CHECK-NEXT: s_mov_b64 s[52:53], exec
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1460 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
; CHECK-NEXT: s_and_b64 s[36:37], s[52:53], s[36:37]
-; CHECK-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr9
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v8, v2
+; CHECK-NEXT: v_mov_b32_e32 v10, v4
; CHECK-NEXT: s_mov_b64 exec, s[36:37]
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.9: ; %cond.load
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[8:9], v[0:1]
; CHECK-NEXT: .LBB0_10: ; %else
; CHECK-NEXT: s_or_b64 exec, exec, s[52:53]
@@ -795,7 +795,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_mov_b64 exec, s[34:35]
; CHECK-NEXT: s_cbranch_execz .LBB0_12
; CHECK-NEXT: ; %bb.11: ; %cond.load1
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[0:1] offset:8
; CHECK-NEXT: .LBB0_12: ; %else2
; CHECK-NEXT: s_or_b64 exec, exec, s[36:37]
@@ -816,7 +815,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_and_saveexec_b64 s[94:95], s[92:93]
; CHECK-NEXT: s_cbranch_execz .LBB0_18
; CHECK-NEXT: ; %bb.17: ; %cond.load10
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[22:23], v[0:1] offset:32
; CHECK-NEXT: .LBB0_18: ; %else11
; CHECK-NEXT: s_or_b64 exec, exec, s[94:95]
@@ -830,7 +828,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_and_saveexec_b64 s[90:91], s[88:89]
; CHECK-NEXT: s_cbranch_execz .LBB0_22
; CHECK-NEXT: ; %bb.21: ; %cond.load16
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[0:1] offset:48
; CHECK-NEXT: .LBB0_22: ; %else17
; CHECK-NEXT: s_or_b64 exec, exec, s[90:91]
@@ -879,7 +876,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_and_saveexec_b64 s[60:61], s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_36
; CHECK-NEXT: ; %bb.35: ; %cond.load37
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:104
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
@@ -890,14 +886,12 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_and_saveexec_b64 s[58:59], s[56:57]
; CHECK-NEXT: s_cbranch_execz .LBB0_38
; CHECK-NEXT: ; %bb.37: ; %cond.load40
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 a[26:27], v[0:1] offset:112
; CHECK-NEXT: .LBB0_38: ; %else41
; CHECK-NEXT: s_or_b64 exec, exec, s[58:59]
; CHECK-NEXT: s_and_saveexec_b64 s[56:57], vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_40
; CHECK-NEXT: ; %bb.39: ; %cond.load43
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_load_dwordx2 a[28:29], v[0:1] offset:120
; CHECK-NEXT: .LBB0_40: ; %else44
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
@@ -1121,81 +1115,52 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_xor_b64 s[46:47], exec, s[58:59]
; CHECK-NEXT: s_cbranch_execz .LBB0_47
; CHECK-NEXT: ; %bb.46: ; %bb1b
-; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
-; CHECK-NEXT: v_accvgpr_read_b32 v32, a30
-; CHECK-NEXT: v_lshrrev_b32_e32 v33, 15, v31
-; CHECK-NEXT: v_alignbit_b32 v36, v31, v32, 15
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1504 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:1508 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1512 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1516 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:1520 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:1524 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1528 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1532 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1536 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1540 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:1544 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:1548 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:1552 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:1556 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:1560 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:1564 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:1568 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:1572 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:1576 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:1580 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:1584 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:1588 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:1592 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:1596 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:1600 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:1604 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:1608 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:1612 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:1616 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:1620 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:1624 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:1628 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1508 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:984 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:988 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v31, a31
+; CHECK-NEXT: v_lshrrev_b32_e32 v2, 15, v31
+; CHECK-NEXT: v_accvgpr_read_b32 v30, a30
+; CHECK-NEXT: v_alignbit_b32 v3, v31, v30, 15
; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v32
+; CHECK-NEXT: v_lshlrev_b32_e32 v6, 17, v30
; CHECK-NEXT: v_and_b32_e32 v59, 0x7fffffff, v59
; CHECK-NEXT: v_and_b32_e32 v51, 0x7fffffff, v51
; CHECK-NEXT: v_and_b32_e32 v57, 0x7fffffff, v57
; CHECK-NEXT: v_and_b32_e32 v49, 0x7fffffff, v49
; CHECK-NEXT: v_and_b32_e32 v43, 0x7fffffff, v43
-; CHECK-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v4, 0x7fffffff, v0
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x7c, v10
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_short v[0:1], v33
+; CHECK-NEXT: flat_store_short v[0:1], v2
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x78, v10
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
-; CHECK-NEXT: flat_store_dword v[0:1], v36
+; CHECK-NEXT: flat_store_dword v[0:1], v3
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:1488 ; 4-byte Folded Reload
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:1492 ; 4-byte Folded Reload
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 31, v42
-; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
; CHECK-NEXT: v_or_b32_e32 v41, v4, v0
-; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v10
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
+; CHECK-NEXT: v_lshrrev_b64 v[42:43], 1, v[42:43]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], 14, v[2:3]
; CHECK-NEXT: v_or_b32_e32 v5, v5, v6
; CHECK-NEXT: v_lshlrev_b32_e32 v6, 18, v2
; CHECK-NEXT: v_lshrrev_b64 v[2:3], 13, v[8:9]
; CHECK-NEXT: v_or_b32_e32 v3, v3, v6
; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:1472 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:1476 ; 4-byte Folded Reload
-; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 24, v10
-; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
; CHECK-NEXT: v_accvgpr_read_b32 v9, a33
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[2:5] offset:80
; CHECK-NEXT: v_and_b32_e32 v9, 0x7fffffff, v9
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 19, v8
; CHECK-NEXT: v_accvgpr_read_b32 v8, a32
-; CHECK-NEXT: v_lshrrev_b64 v[42:43], 1, v[42:43]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_and_b32_e32 v7, 0x7fffffff, v7
; CHECK-NEXT: v_lshrrev_b64 v[4:5], 12, v[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir b/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
index 112402ca38e2a..6dab1a84f5242 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-nolivesubranges.mir
@@ -27,8 +27,8 @@ body: |
; CHECK-NEXT: renamable $sgpr1 = IMPLICIT_DEF
; CHECK-NEXT: SI_SPILL_S64_SAVE renamable $sgpr0_sgpr1, %stack.0, implicit $exec, implicit $sp_reg :: (store (s64) into %stack.0, align 4, addrspace 5)
; CHECK-NEXT: KILL implicit-def $vcc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr32_sgpr33_sgpr34_sgpr35_sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit-def $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit-def $sgpr96_sgpr97_sgpr98_sgpr99_sgpr100_sgpr101_sgpr102_sgpr103
- ; CHECK-NEXT: renamable $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.0, 0, implicit $exec, implicit $sp_reg :: (load (s64) from %stack.0, align 4, addrspace 5)
- ; CHECK-NEXT: $sgpr105 = S_AND_B32 killed renamable $sgpr1, renamable $sgpr1, implicit-def $scc
+ ; CHECK-NEXT: renamable $sgpr0 = SI_SPILL_S32_RESTORE %stack.0, 1, implicit $exec, implicit $sp_reg :: (load (s32) from %stack.0, addrspace 5)
+ ; CHECK-NEXT: $sgpr105 = S_AND_B32 killed renamable $sgpr0, renamable $sgpr0, implicit-def $scc
; CHECK-NEXT: S_NOP 0, implicit $sgpr104, implicit $sgpr105
%0:sreg_64 = COPY $sgpr0_sgpr1
%0.sub0:sreg_64 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
index 2ef6908d3701d..c41a6e52072c0 100644
--- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
+++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir
@@ -34,20 +34,16 @@ body: |
; CHECK-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3:0x00000000000000FF, $vgpr4_vgpr5_vgpr6_vgpr7:0x00000000000000FF
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: SI_SPILL_AV128_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3, %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr0 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr6, 0, killed $vgpr0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: renamable $vgpr0 = V_TRUNC_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
- ; CHECK-NEXT: SI_SPILL_AV32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr7 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, $vgpr7, 0, killed $vgpr3, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr6 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: renamable $vgpr2_vgpr3 = lr-split COPY killed renamable $vgpr6_vgpr7
- ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5)
- ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF
- ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, $vgpr4, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
+ ; CHECK-NEXT: renamable $vgpr2 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 12, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr3 = nofpexcept V_DIV_FIXUP_F32_e64 0, $vgpr1, 0, $vgpr7, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr2 = SI_SPILL_AV32_RESTORE %stack.2, $sgpr32, 8, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr2 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6_vgpr7 = SI_SPILL_AV64_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.2, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr1, 0, $vgpr4, 0, $vgpr6, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $vgpr1 = V_FMA_F32_e64 0, killed $vgpr5, 0, $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: dead renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SCRATCH_LOAD_DWORDX4_SADDR %stack.0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load (s128), addrspace 5)
; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll b/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
deleted file mode 100644
index f09bfe95b9b0e..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; RUN: not llc -mtriple=amdgpu11.00-amd-amdhsa -stress-regalloc=4 -amdgpu-enable-rewrite-partial-reg-uses=0 -filetype=null -verify-machineinstrs %s 2>&1 | FileCheck %s
-
-; CHECK: error: <unknown>:0:0: ran out of registers during register allocation in function 'f'
-; CHECK-NOT: Bad machine code
-
-define <16 x half> @f(i1 %LGV2, <16 x half> %0) {
-BB:
- br i1 %LGV2, label %SW_C3, label %SW_C
-
-SW_C: ; preds = %BB
- %B1 = fmul <16 x half> %0, zeroinitializer
- ret <16 x half> %B1
-
-SW_C3: ; preds = %BB
- ret <16 x half> <half 0xH0000, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison, half poison>
-}
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
index 735b8e0f74089..ebc023d5c157e 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-mark-last-scratch-load.ll
@@ -295,27 +295,25 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b32 s0, s33
; CHECK-NEXT: s_mov_b32 s33, s32
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
-; CHECK-NEXT: scratch_store_b32 off, v93, s33 offset:404 ; 4-byte Folded Spill
+; CHECK-NEXT: scratch_store_b32 off, v47, s33 offset:400 ; 4-byte Folded Spill
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_mov_b32 exec_lo, s1
-; CHECK-NEXT: v_writelane_b32 v93, s0, 14
+; CHECK-NEXT: v_writelane_b32 v47, s0, 14
; CHECK-NEXT: s_addk_co_i32 s32, 0x1a0
; CHECK-NEXT: s_clause 0x1f ; 128-byte Folded Spill
-; CHECK-NEXT: scratch_store_b32 off, v40, s33 offset:144
+; CHECK-NEXT: scratch_store_b32 off, v40, s33 offset:140
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v41, s33 offset:140
+; CHECK-NEXT: scratch_store_b32 off, v41, s33 offset:136
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v42, s33 offset:136
+; CHECK-NEXT: scratch_store_b32 off, v42, s33 offset:132
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v43, s33 offset:132
+; CHECK-NEXT: scratch_store_b32 off, v43, s33 offset:128
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v44, s33 offset:128
+; CHECK-NEXT: scratch_store_b32 off, v44, s33 offset:124
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v45, s33 offset:124
+; CHECK-NEXT: scratch_store_b32 off, v45, s33 offset:120
; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v46, s33 offset:120
-; CHECK-NEXT: ; meta instruction
-; CHECK-NEXT: scratch_store_b32 off, v47, s33 offset:116
+; CHECK-NEXT: scratch_store_b32 off, v46, s33 offset:116
; CHECK-NEXT: ; meta instruction
; CHECK-NEXT: scratch_store_b32 off, v56, s33 offset:112
; CHECK-NEXT: ; meta instruction
@@ -364,9 +362,9 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: scratch_store_b32 off, v105, s33 offset:24
; CHECK-NEXT: ; meta instruction
; CHECK-NEXT: scratch_store_b32 off, v106, s33 offset:20
-; CHECK-NEXT: s_clause 0x4 ; 20-byte Folded Spill
-; CHECK-NEXT: scratch_store_b32 off, v107, s33 offset:16
; CHECK-NEXT: ; meta instruction
+; CHECK-NEXT: scratch_store_b32 off, v107, s33 offset:16
+; CHECK-NEXT: s_clause 0x3 ; 16-byte Folded Spill
; CHECK-NEXT: scratch_store_b32 off, v108, s33 offset:12
; CHECK-NEXT: ; meta instruction
; CHECK-NEXT: scratch_store_b32 off, v109, s33 offset:8
@@ -374,20 +372,20 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: scratch_store_b32 off, v110, s33 offset:4
; CHECK-NEXT: ; meta instruction
; CHECK-NEXT: scratch_store_b32 off, v111, s33
-; CHECK-NEXT: v_writelane_b32 v93, s34, 0
-; CHECK-NEXT: v_writelane_b32 v93, s35, 1
-; CHECK-NEXT: v_writelane_b32 v93, s36, 2
-; CHECK-NEXT: v_writelane_b32 v93, s37, 3
-; CHECK-NEXT: v_writelane_b32 v93, s38, 4
-; CHECK-NEXT: v_writelane_b32 v93, s39, 5
-; CHECK-NEXT: v_writelane_b32 v93, s48, 6
-; CHECK-NEXT: v_writelane_b32 v93, s49, 7
-; CHECK-NEXT: v_writelane_b32 v93, s50, 8
-; CHECK-NEXT: v_writelane_b32 v93, s51, 9
-; CHECK-NEXT: v_writelane_b32 v93, s52, 10
-; CHECK-NEXT: v_writelane_b32 v93, s53, 11
-; CHECK-NEXT: v_writelane_b32 v93, s30, 12
-; CHECK-NEXT: v_writelane_b32 v93, s31, 13
+; CHECK-NEXT: v_writelane_b32 v47, s34, 0
+; CHECK-NEXT: v_writelane_b32 v47, s35, 1
+; CHECK-NEXT: v_writelane_b32 v47, s36, 2
+; CHECK-NEXT: v_writelane_b32 v47, s37, 3
+; CHECK-NEXT: v_writelane_b32 v47, s38, 4
+; CHECK-NEXT: v_writelane_b32 v47, s39, 5
+; CHECK-NEXT: v_writelane_b32 v47, s48, 6
+; CHECK-NEXT: v_writelane_b32 v47, s49, 7
+; CHECK-NEXT: v_writelane_b32 v47, s50, 8
+; CHECK-NEXT: v_writelane_b32 v47, s51, 9
+; CHECK-NEXT: v_writelane_b32 v47, s52, 10
+; CHECK-NEXT: v_writelane_b32 v47, s53, 11
+; CHECK-NEXT: v_writelane_b32 v47, s30, 12
+; CHECK-NEXT: v_writelane_b32 v47, s31, 13
; CHECK-NEXT: v_dual_mov_b32 v92, v31 :: v_dual_mov_b32 v1, 0
; CHECK-NEXT: v_dual_mov_b32 v0, 0x60 :: v_dual_mov_b32 v3, 0
; CHECK-NEXT: v_dual_mov_b32 v2, 0x50 :: v_dual_mov_b32 v5, 0
@@ -430,18 +428,18 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b64 s[38:39], s[6:7]
; CHECK-NEXT: s_mov_b64 s[48:49], s[4:5]
; CHECK-NEXT: s_wait_loadcnt 0x5
-; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:148 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:144 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x4
-; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:164 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:160 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x3
-; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:180 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:176 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x2
; CHECK-NEXT: s_clause 0x4 ; 80-byte Folded Spill
-; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:196
-; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:212
-; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:228
-; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:244
-; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:260
+; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:192
+; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:208
+; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:224
+; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:240
+; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:256
; CHECK-NEXT: s_clause 0x4
; CHECK-NEXT: global_load_b128 v[10:13], v[8:9], off offset:48
; CHECK-NEXT: global_load_b128 v[14:17], v[8:9], off offset:64
@@ -449,31 +447,31 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: global_load_b128 v[22:25], v[8:9], off offset:96
; CHECK-NEXT: global_load_b128 v[26:29], v[8:9], off offset:112
; CHECK-NEXT: s_wait_loadcnt 0x4
-; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:276 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[10:13], s33 offset:272 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x3
-; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:292 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[14:17], s33 offset:288 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:308 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[18:21], s33 offset:304 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x1
-; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:324 ; 16-byte Folded Spill
+; CHECK-NEXT: scratch_store_b128 off, v[22:25], s33 offset:320 ; 16-byte Folded Spill
; CHECK-NEXT: s_wait_loadcnt 0x0
; CHECK-NEXT: s_clause 0x3 ; 64-byte Folded Spill
-; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:340
-; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:356
-; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:372
-; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:388
+; CHECK-NEXT: scratch_store_b128 off, v[26:29], s33 offset:336
+; CHECK-NEXT: scratch_store_b128 off, v[30:33], s33 offset:352
+; CHECK-NEXT: scratch_store_b128 off, v[34:37], s33 offset:368
+; CHECK-NEXT: scratch_store_b128 off, v[38:41], s33 offset:384
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
; CHECK-NEXT: s_clause 0x7 ; 128-byte Folded Reload
-; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:276 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:292 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:308 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:324 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:340 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:356 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:372 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:388 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[0:3], off, s33 offset:272 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[4:7], off, s33 offset:288 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:304 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[12:15], off, s33 offset:320 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[16:19], off, s33 offset:336 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[20:23], off, s33 offset:352 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[24:27], off, s33 offset:368 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[28:31], off, s33 offset:384 th:TH_LOAD_LU
; CHECK-NEXT: s_getpc_b64 s[0:1]
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_sext_i32_i16 s1, s1
@@ -489,38 +487,35 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: s_mov_b32 s13, s52
; CHECK-NEXT: s_mov_b32 s14, s51
; CHECK-NEXT: s_mov_b32 s15, s50
+; CHECK-NEXT: s_wait_loadcnt 0x7
+; CHECK-NEXT: v_dual_mov_b32 v44, v0 :: v_dual_mov_b32 v45, v1
+; CHECK-NEXT: v_mov_b32_e32 v46, v2
+; CHECK-NEXT: v_mov_b32_e32 v0, v72
; CHECK-NEXT: s_wait_loadcnt 0x3
; CHECK-NEXT: scratch_store_b32 off, v19, s32 offset:128
; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_load_b128 v[19:22], off, s33 offset:148 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: scratch_load_b128 v[19:22], off, s33 offset:144 th:TH_LOAD_LU ; 16-byte Folded Reload
; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: scratch_load_b128 v[23:26], off, s33 offset:164 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: scratch_load_b128 v[23:26], off, s33 offset:160 th:TH_LOAD_LU ; 16-byte Folded Reload
; CHECK-NEXT: s_wait_loadcnt 0x2
-; CHECK-NEXT: s_clause 0x5 ; 96-byte Folded Reload
-; CHECK-NEXT: scratch_load_b128 v[27:30], off, s33 offset:180 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[31:34], off, s33 offset:196 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[35:38], off, s33 offset:212 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[39:42], off, s33 offset:228 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[43:46], off, s33 offset:244 th:TH_LOAD_LU
-; CHECK-NEXT: scratch_load_b128 v[47:50], off, s33 offset:260 th:TH_LOAD_LU
+; CHECK-NEXT: s_clause 0x1 ; 32-byte Folded Reload
+; CHECK-NEXT: scratch_load_b128 v[27:30], off, s33 offset:176 th:TH_LOAD_LU
+; CHECK-NEXT: scratch_load_b128 v[31:34], off, s33 offset:192 th:TH_LOAD_LU
; CHECK-NEXT: s_clause 0x3
; CHECK-NEXT: scratch_store_b128 off, v[15:18], s32 offset:112
; CHECK-NEXT: scratch_store_b128 off, v[11:14], s32 offset:96
; CHECK-NEXT: scratch_store_b128 off, v[7:10], s32 offset:80
; CHECK-NEXT: scratch_store_b128 off, v[3:6], s32 offset:64
-; CHECK-NEXT: v_dual_mov_b32 v4, v88 :: v_dual_mov_b32 v5, v89
-; CHECK-NEXT: v_dual_mov_b32 v6, v90 :: v_dual_mov_b32 v7, v91
-; CHECK-NEXT: v_dual_mov_b32 v8, v76 :: v_dual_mov_b32 v9, v77
-; CHECK-NEXT: v_dual_mov_b32 v10, v78 :: v_dual_mov_b32 v11, v79
-; CHECK-NEXT: v_dual_mov_b32 v12, v60 :: v_dual_mov_b32 v13, v61
-; CHECK-NEXT: v_dual_mov_b32 v14, v62 :: v_dual_mov_b32 v15, v63
-; CHECK-NEXT: v_dual_mov_b32 v16, v108 :: v_dual_mov_b32 v17, v109
-; CHECK-NEXT: v_mov_b32_e32 v18, v110
-; CHECK-NEXT: s_wait_loadcnt 0x1
-; CHECK-NEXT: v_dual_mov_b32 v44, v0 :: v_dual_mov_b32 v45, v1
-; CHECK-NEXT: v_mov_b32_e32 v46, v2
-; CHECK-NEXT: v_dual_mov_b32 v0, v72 :: v_dual_mov_b32 v1, v73
-; CHECK-NEXT: v_dual_mov_b32 v2, v74 :: v_dual_mov_b32 v3, v75
+; CHECK-NEXT: v_dual_mov_b32 v1, v73 :: v_dual_mov_b32 v2, v74
+; CHECK-NEXT: v_dual_mov_b32 v3, v75 :: v_dual_mov_b32 v4, v88
+; CHECK-NEXT: v_dual_mov_b32 v5, v89 :: v_dual_mov_b32 v6, v90
+; CHECK-NEXT: v_dual_mov_b32 v7, v91 :: v_dual_mov_b32 v8, v76
+; CHECK-NEXT: v_dual_mov_b32 v9, v77 :: v_dual_mov_b32 v10, v78
+; CHECK-NEXT: v_dual_mov_b32 v11, v79 :: v_dual_mov_b32 v12, v60
+; CHECK-NEXT: v_dual_mov_b32 v13, v61 :: v_dual_mov_b32 v14, v62
+; CHECK-NEXT: v_dual_mov_b32 v15, v63 :: v_dual_mov_b32 v16, v108
+; CHECK-NEXT: v_dual_mov_b32 v17, v109 :: v_dual_mov_b32 v18, v110
+; CHECK-NEXT: s_wait_loadcnt 0x0
; CHECK-NEXT: v_dual_mov_b32 v43, v34 :: v_dual_mov_b32 v42, v33
; CHECK-NEXT: v_dual_mov_b32 v41, v32 :: v_dual_mov_b32 v40, v31
; CHECK-NEXT: v_dual_mov_b32 v39, v30 :: v_dual_mov_b32 v38, v29
@@ -576,33 +571,32 @@ define <8 x half> @baz() nounwind {
; CHECK-NEXT: scratch_load_b32 v58, off, s33 offset:104
; CHECK-NEXT: scratch_load_b32 v57, off, s33 offset:108
; CHECK-NEXT: scratch_load_b32 v56, off, s33 offset:112
-; CHECK-NEXT: scratch_load_b32 v47, off, s33 offset:116
-; CHECK-NEXT: scratch_load_b32 v46, off, s33 offset:120
-; CHECK-NEXT: scratch_load_b32 v45, off, s33 offset:124
-; CHECK-NEXT: s_clause 0x4 ; 20-byte Folded Reload
-; CHECK-NEXT: scratch_load_b32 v44, off, s33 offset:128
-; CHECK-NEXT: scratch_load_b32 v43, off, s33 offset:132
-; CHECK-NEXT: scratch_load_b32 v42, off, s33 offset:136
-; CHECK-NEXT: scratch_load_b32 v41, off, s33 offset:140
-; CHECK-NEXT: scratch_load_b32 v40, off, s33 offset:144
-; CHECK-NEXT: v_readlane_b32 s30, v93, 12
-; CHECK-NEXT: v_readlane_b32 s31, v93, 13
-; CHECK-NEXT: v_readlane_b32 s53, v93, 11
-; CHECK-NEXT: v_readlane_b32 s52, v93, 10
-; CHECK-NEXT: v_readlane_b32 s51, v93, 9
-; CHECK-NEXT: v_readlane_b32 s50, v93, 8
-; CHECK-NEXT: v_readlane_b32 s49, v93, 7
-; CHECK-NEXT: v_readlane_b32 s48, v93, 6
-; CHECK-NEXT: v_readlane_b32 s39, v93, 5
-; CHECK-NEXT: v_readlane_b32 s38, v93, 4
-; CHECK-NEXT: v_readlane_b32 s37, v93, 3
-; CHECK-NEXT: v_readlane_b32 s36, v93, 2
-; CHECK-NEXT: v_readlane_b32 s35, v93, 1
-; CHECK-NEXT: v_readlane_b32 s34, v93, 0
+; CHECK-NEXT: scratch_load_b32 v46, off, s33 offset:116
+; CHECK-NEXT: scratch_load_b32 v45, off, s33 offset:120
+; CHECK-NEXT: scratch_load_b32 v44, off, s33 offset:124
+; CHECK-NEXT: s_clause 0x3 ; 16-byte Folded Reload
+; CHECK-NEXT: scratch_load_b32 v43, off, s33 offset:128
+; CHECK-NEXT: scratch_load_b32 v42, off, s33 offset:132
+; CHECK-NEXT: scratch_load_b32 v41, off, s33 offset:136
+; CHECK-NEXT: scratch_load_b32 v40, off, s33 offset:140
+; CHECK-NEXT: v_readlane_b32 s30, v47, 12
+; CHECK-NEXT: v_readlane_b32 s31, v47, 13
+; CHECK-NEXT: v_readlane_b32 s53, v47, 11
+; CHECK-NEXT: v_readlane_b32 s52, v47, 10
+; CHECK-NEXT: v_readlane_b32 s51, v47, 9
+; CHECK-NEXT: v_readlane_b32 s50, v47, 8
+; CHECK-NEXT: v_readlane_b32 s49, v47, 7
+; CHECK-NEXT: v_readlane_b32 s48, v47, 6
+; CHECK-NEXT: v_readlane_b32 s39, v47, 5
+; CHECK-NEXT: v_readlane_b32 s38, v47, 4
+; CHECK-NEXT: v_readlane_b32 s37, v47, 3
+; CHECK-NEXT: v_readlane_b32 s36, v47, 2
+; CHECK-NEXT: v_readlane_b32 s35, v47, 1
+; CHECK-NEXT: v_readlane_b32 s34, v47, 0
; CHECK-NEXT: s_mov_b32 s32, s33
-; CHECK-NEXT: v_readlane_b32 s0, v93, 14
+; CHECK-NEXT: v_readlane_b32 s0, v47, 14
; CHECK-NEXT: s_or_saveexec_b32 s1, -1
-; CHECK-NEXT: scratch_load_b32 v93, off, s33 offset:404 ; 4-byte Folded Reload
+; CHECK-NEXT: scratch_load_b32 v47, off, s33 offset:400 ; 4-byte Folded Reload
; CHECK-NEXT: s_wait_alu depctr_sa_sdst(0)
; CHECK-NEXT: s_mov_b32 exec_lo, s1
; CHECK-NEXT: s_mov_b32 s33, s0
>From 829c8e9360ece47f2ec76c67ee0f78306e2312cd Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Fri, 31 Jul 2026 14:47:42 +0530
Subject: [PATCH 2/2] Fix merge conflicts while rebasing on main.
---
llvm/include/llvm/CodeGen/TargetRegisterInfo.h | 5 -----
llvm/lib/CodeGen/InlineSpiller.cpp | 4 ----
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 5 -----
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 3 ---
4 files changed, 17 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 38ffe35010379..85fc6f7c92fdc 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1088,11 +1088,6 @@ class LLVM_ABI TargetRegisterInfo : public MCRegisterInfo {
bool checkAllSuperRegsMarked(const BitVector &RegisterSet,
ArrayRef<MCPhysReg> Exceptions = ArrayRef<MCPhysReg>()) const;
- virtual const TargetRegisterClass *
- getConstrainedRegClass(const TargetRegisterClass *RC) const {
- return RC;
- }
-
virtual const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 6deab6fefb2a3..1e92ef14813f5 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1418,10 +1418,6 @@ void InlineSpiller::spillAroundUses(Register Reg) {
const TargetRegisterClass *NewRC =
SubReg ? TRI.getSubRegisterClass(OrigRC, SubReg) : nullptr;
- // Check if the target needs to constrain the RC further.
- if (NewRC)
- NewRC = TRI.getConstrainedRegClass(NewRC);
-
Register NewVReg = Edit->createFrom(Reg, NewRC);
if (RI.Reads)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index c8974cbf4885b..46e9af67115af 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4010,11 +4010,6 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
}
}
-const TargetRegisterClass *
-SIRegisterInfo::getConstrainedRegClass(const TargetRegisterClass *RC) const {
- return getProperlyAlignedRC(RC);
-}
-
const TargetRegisterClass *
SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 1ae4594632397..108e5a24f50d2 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -378,9 +378,6 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
return getRegClassForSizeOnBank(Ty.getSizeInBits(), Bank);
}
- const TargetRegisterClass *
- getConstrainedRegClass(const TargetRegisterClass *RC) const override;
-
const TargetRegisterClass *
getConstrainedRegClassForOperand(const MachineOperand &MO,
const MachineRegisterInfo &MRI) const override;
More information about the llvm-branch-commits
mailing list